1
0
mirror of https://github.com/strongdm/comply synced 2024-12-25 15:01:35 +00:00

Merge pull request #108 from camposer/fix/infinite-build

Fix infinite build, pandoc always pulling and building files with strikethrough
This commit is contained in:
Rodolfo Campos 2021-11-03 15:37:39 +01:00 committed by GitHub
commit 9bcac77e52
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 2201 additions and 555 deletions

1
.env.example Normal file
View File

@ -0,0 +1 @@
COMPLY_USE_LOCAL_PANDOC=

4
.gitignore vendored
View File

@ -3,4 +3,6 @@ output
dist
.envrc
bindata.go
.idea/
.idea/
.env
.vscode/

View File

@ -1,4 +1,4 @@
FROM strongdm/pandoc:latest
FROM strongdm/pandoc:edge
# based on implementation by James Gregory <james@jagregory.com>
MAINTAINER Comply <comply@strongdm.com>

View File

@ -52,8 +52,8 @@ else
endif
docker:
cd build && docker build -t strongdm/pandoc .
docker push strongdm/pandoc
cd build && docker build -t strongdm/pandoc:edge .
docker push strongdm/pandoc:edge
cleanse:
git checkout --orphan newbranch

View File

@ -1,29 +1,10 @@
FROM haskell:latest
FROM pandoc/ubuntu
# based on implementation by James Gregory <james@jagregory.com>
MAINTAINER Comply <comply@strongdm.com>
# install latex packages
RUN apt-get update -y \
&& apt-get install -y -o Acquire::Retries=10 --no-install-recommends \
texlive-latex-base \
texlive-xetex \
texlive-fonts-recommended \
latex-xcolor \
texlive-latex-extra \
fontconfig \
unzip \
lmodern
# will ease up the update process
# updating this env variable will trigger the automatic build of the Docker image
ENV PANDOC_VERSION "2.2.1"
# install pandoc
RUN cabal update && cabal install pandoc-${PANDOC_VERSION}
&& DEBIAN_FRONTEND=noninteractive apt-get install -y -o Acquire::Retries=10 --no-install-recommends \
texlive-latex-base \
texlive-latex-extra \
texlive-plain-generic \
lmodern
WORKDIR /source
ENTRYPOINT ["/root/.cabal/bin/pandoc"]
CMD ["--help"]

View File

@ -1,7 +1,17 @@
package main
import "github.com/strongdm/comply/internal/cli"
import (
"fmt"
"path/filepath"
"runtime"
"github.com/joho/godotenv"
"github.com/strongdm/comply/internal/cli"
)
func main() {
_, b, _, _ := runtime.Caller(0)
basepath := filepath.Dir(b)
godotenv.Load(fmt.Sprintf("%s/.env", basepath))
cli.Main()
}

File diff suppressed because it is too large Load Diff

View File

@ -91,4 +91,4 @@ TODO: Finalize these lists
## External
{{.Name}} communicates relevant control-related information to external parties including shareholders, customers, contractors, regulators, and government entities as needed according to contractual and regulatory/statutory obligation.
{{.Name}} communicates relevant control-related information to external parties including shareholders, customers, contractors, regulators, and government entities as needed according to contractual and regulatory/statutory obligation.

View File

@ -53,5 +53,3 @@ a. *When an employee changes roles within the organization:*
a. *Review of accounts and permissions:*
i. Each month, IT and HR will review accounts and permission levels for accuracy.

1
go.mod
View File

@ -25,6 +25,7 @@ require (
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-retryablehttp v0.7.0 // indirect
github.com/jcelliott/lumber v0.0.0-20160324203708-dd349441af25 // indirect
github.com/joho/godotenv v1.4.0
github.com/juju/ansiterm v0.0.0-20210929141451-8b71cc96ebdc // indirect
github.com/kevinburke/rest v0.0.0-20210506044642-5611499aa33c // indirect
github.com/manifoldco/promptui v0.8.0

2
go.sum
View File

@ -598,6 +598,8 @@ github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeY
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
github.com/joho/godotenv v1.4.0 h1:3l4+N6zfMWnkbPEXKng2o2/MR5mSwTrBih4ZEkkz1lg=
github.com/joho/godotenv v1.4.0/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

View File

@ -137,10 +137,10 @@ func notifyVersion(c *cli.Context) error {
func pandocMustExist(c *cli.Context) error {
eitherMustExistErr := fmt.Errorf("\n\nPlease install either Docker or the pandoc package and re-run `%s`. Find OS-specific pandoc installation instructions at: https://pandoc.org/installing.html", c.Command.Name)
pandocExistErr, found, goodVersion, pdfLatex := pandocBinaryMustExist(c)
pandocBinaryExistErr, found, goodVersion, pdfLatex := pandocBinaryMustExist(c)
dockerExistErr, inPath, isRunning := dockerMustExist(c)
config.SetPandoc(pandocExistErr == nil, dockerExistErr == nil)
config.SetPandoc(pandocBinaryExistErr == nil, dockerExistErr == nil)
check := func(b bool) string {
if b {
return "✔"
@ -150,7 +150,7 @@ func pandocMustExist(c *cli.Context) error {
}
if pandocExistErr != nil && dockerExistErr != nil {
if pandocBinaryExistErr != nil && dockerExistErr != nil {
fmt.Printf(`
[%s] pandoc binary installed and in PATH
@ -165,14 +165,20 @@ func pandocMustExist(c *cli.Context) error {
}
// if we don't have pandoc, but we do have docker, execute a pull
if (pandocExistErr != nil && dockerExistErr == nil) || config.WhichPandoc() == config.UseDocker {
dockerPull(c)
if !pandocImageExists(context.Background()) && ((pandocBinaryExistErr != nil && dockerExistErr == nil) || config.WhichPandoc() == config.UseDocker) {
canPullPandoc := strings.TrimSpace(strings.ToLower(os.Getenv("COMPLY_USE_LOCAL_PANDOC"))) != "true"
if canPullPandoc {
fmt.Println("Pulling docker image")
dockerPull(c)
} else {
return fmt.Errorf("Local Pandoc not found. Please set COMPLY_USE_LOCAL_PANDOC to false")
}
}
return nil
}
func pandocBinaryMustExist(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
var pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
cmd := exec.Command("pandoc", "-v")
outputRaw, err := cmd.Output()
@ -230,7 +236,7 @@ func pandocBinaryMustExist(c *cli.Context) (e error, found, goodVersion, pdfLate
return e, found, goodVersion, pdfLatex
}
func dockerMustExist(c *cli.Context) (e error, inPath, isRunning bool) {
var dockerMustExist = func(c *cli.Context) (e error, inPath, isRunning bool) {
dockerErr := fmt.Errorf("Docker must be available in order to run `%s`", c.Command.Name)
inPath = true
@ -257,7 +263,26 @@ func dockerMustExist(c *cli.Context) (e error, inPath, isRunning bool) {
return nil, inPath, isRunning
}
func dockerPull(c *cli.Context) error {
var pandocImageExists = func(ctx context.Context) bool {
cli, err := client.NewEnvClient()
if err != nil {
return false
}
options := types.ImageListOptions{All: true}
imageList, err := cli.ImageList(ctx, options)
if err != nil {
return false
}
for _, image := range imageList {
if len(image.RepoTags) > 0 && strings.Contains(image.RepoTags[0], "strongdm/pandoc:edge") {
return true
}
}
return false
}
var dockerPull = func(c *cli.Context) error {
dockerErr := fmt.Errorf("Docker must be available in order to run `%s`", c.Command.Name)
ctx := context.Background()
@ -275,7 +300,7 @@ func dockerPull(c *cli.Context) error {
select {
case <-longishPull:
fmt.Print("Pulling strongdm/pandoc:latest Docker image (this will take some time) ")
fmt.Print("Pulling strongdm/pandoc:edge Docker image (this will take some time) ")
go func() {
for {
@ -294,7 +319,7 @@ func dockerPull(c *cli.Context) error {
}
}()
r, err := cli.ImagePull(ctx, "strongdm/pandoc:latest", types.ImagePullOptions{})
r, err := cli.ImagePull(ctx, "strongdm/pandoc:edge", types.ImagePullOptions{})
if err != nil {
return dockerErr
}
@ -327,7 +352,7 @@ func cleanContainers(c *cli.Context) error {
for _, c := range containers {
// assume this container was leftover from previous aborted run
if strings.HasPrefix(c.Image, "strongdm/pandoc") {
if strings.HasPrefix(c.Image, "strongdm/pandoc:edge") {
d := time.Second * 2
err = cli.ContainerStop(ctx, c.ID, &d)
if err != nil {

189
internal/cli/app_test.go Normal file
View File

@ -0,0 +1,189 @@
package cli
import (
"context"
"errors"
"os"
"reflect"
"testing"
"github.com/strongdm/comply/internal/util"
"github.com/urfave/cli"
)
type PandocMustExist struct{}
func TestPandocMustExist(t *testing.T) {
util.ExecuteTests(t, reflect.TypeOf(PandocMustExist{}), beforeEach, nil)
}
func beforeEach() {
util.MockConfig()
}
func (tg PandocMustExist) WhenBinaryExists(t *testing.T) {
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return nil, true, true, true
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return errors.New("docker doesn't exist"), false, false
}
pandocImageExists = func(ctx context.Context) bool {
return false
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
}
func (tg PandocMustExist) WhenImageExists(t *testing.T) {
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, true, true
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return errors.New("docker doesn't exist"), false, false
}
pandocImageExists = func(ctx context.Context) bool {
return true
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
}
func (tg PandocMustExist) WhenBinaryAndImageDontExists(t *testing.T) {
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, false, false
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return nil, true, false
}
pandocImageExists = func(ctx context.Context) bool {
return false
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != true {
t.Fatal("Docker wasn't pulled")
}
}
func (tg PandocMustExist) WhenCannotPullPandoc(t *testing.T) {
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, false, false
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return errors.New("docker doesn't exist"), false, false
}
pandocImageExists = func(ctx context.Context) bool {
return false
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
}
func (tg PandocMustExist) WhenMustUseLocalPandoc(t *testing.T) {
os.Setenv("COMPLY_USE_LOCAL_PANDOC", "true")
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, false, false
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return errors.New("docker doesn't exist"), false, false
}
pandocImageExists = func(ctx context.Context) bool {
return true
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
os.Clearenv()
}
func (tg PandocMustExist) WhenPandocDontExistsAndCannotPull(t *testing.T) {
os.Setenv("COMPLY_USE_LOCAL_PANDOC", "true")
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, false, false
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return nil, true, false
}
pandocImageExists = func(ctx context.Context) bool {
return false
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
os.Clearenv()
}

View File

@ -189,7 +189,6 @@ func loadMDMD(path string) (*metadataMarkdown, error) {
if err != nil {
panic(err)
}
content := string(bytes)
components := strings.Split(content, "---")
if components[0] == "" && (len(components) > 1) {

View File

@ -2,21 +2,27 @@ package model
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"runtime"
"reflect"
"testing"
"github.com/strongdm/comply/internal/config"
"github.com/strongdm/comply/internal/path"
"gopkg.in/yaml.v2"
"github.com/strongdm/comply/internal/util"
)
type ReadFiles struct{}
func beforeEach() {
util.MockConfig()
}
func TestReadFiles(t *testing.T) {
util.ExecuteTests(t, reflect.TypeOf(ReadFiles{}), beforeEach, nil)
}
// TestReadNarratives calls model.ReadNarratives checking for a valid return value.
func TestReadNarratives(t *testing.T) {
mockConfig()
filePath := fmt.Sprintf("%s/narratives/control.md", getRootPath())
func (tg ReadFiles) TestReadNarratives(t *testing.T) {
filePath := fmt.Sprintf("%s/narratives/control.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
path.Narratives = func() ([]path.File, error) {
return []path.File{
@ -38,8 +44,7 @@ func TestReadNarratives(t *testing.T) {
// TestReadNarrativesWhenThereAreNoNarratives calls model.ReadNarratives checking for a valid return when
// there are no narratives to process
func TestReadNarrativesWhenThereAreNoNarratives(t *testing.T) {
mockConfig()
func (tg ReadFiles) TestReadNarrativesWhenThereAreNoNarratives(t *testing.T) {
path.Narratives = func() ([]path.File, error) {
return []path.File{}, nil
}
@ -55,10 +60,9 @@ func TestReadNarrativesWhenThereAreNoNarratives(t *testing.T) {
// TestReadNarrativesFailsWhenInvalidNarrative calls model.ReadNarratives checking for an error return when
// there is an invalid narrative
func TestReadNarrativesFailsWhenInvalidNarrative(t *testing.T) {
mockConfig()
func (tg ReadFiles) TestReadNarrativesFailsWhenInvalidNarrative(t *testing.T) {
path.Narratives = func() ([]path.File, error) {
filePath := fmt.Sprintf("%s/../fixtures/narratives/invalid-control.md", getRootPath())
filePath := fmt.Sprintf("%s/../fixtures/narratives/invalid-control.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
return []path.File{
{FullPath: filePath, Info: fileInfo},
@ -72,9 +76,8 @@ func TestReadNarrativesFailsWhenInvalidNarrative(t *testing.T) {
}
// TestReadProcedures calls model.ReadProcedures checking for a valid return value.
func TestReadProcedures(t *testing.T) {
mockConfig()
filePath := fmt.Sprintf("%s/procedures/workstation.md", getRootPath())
func (tg ReadFiles) TestReadProcedures(t *testing.T) {
filePath := fmt.Sprintf("%s/procedures/workstation.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
path.Procedures = func() ([]path.File, error) {
return []path.File{
@ -96,8 +99,7 @@ func TestReadProcedures(t *testing.T) {
// TestReadProceduresWhenThereAreNoProcedures calls model.ReadProcedures checking for a valid return when
// there are no procedures to process
func TestReadProceduresWhenThereAreNoProcedures(t *testing.T) {
mockConfig()
func (tg ReadFiles) TestReadProceduresWhenThereAreNoProcedures(t *testing.T) {
path.Procedures = func() ([]path.File, error) {
return []path.File{}, nil
}
@ -113,10 +115,9 @@ func TestReadProceduresWhenThereAreNoProcedures(t *testing.T) {
// TestReadProceduresFailsWhenInvalidProcedure calls model.ReadProcedures checking for an error return when
// there is an invalid procedure
func TestReadProceduresFailsWhenInvalidProcedure(t *testing.T) {
mockConfig()
func (tg ReadFiles) TestReadProceduresFailsWhenInvalidProcedure(t *testing.T) {
path.Procedures = func() ([]path.File, error) {
filePath := fmt.Sprintf("%s/../fixtures/procedures/invalid-workstation.md", getRootPath())
filePath := fmt.Sprintf("%s/../fixtures/procedures/invalid-workstation.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
return []path.File{
{FullPath: filePath, Info: fileInfo},
@ -130,9 +131,8 @@ func TestReadProceduresFailsWhenInvalidProcedure(t *testing.T) {
}
// TestReadPolicies calls model.ReadPolicies checking for a valid return value.
func TestReadPolicies(t *testing.T) {
mockConfig()
filePath := fmt.Sprintf("%s/policies/access.md", getRootPath())
func (tg ReadFiles) TestReadPolicies(t *testing.T) {
filePath := fmt.Sprintf("%s/policies/access.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
path.Policies = func() ([]path.File, error) {
return []path.File{
@ -154,8 +154,7 @@ func TestReadPolicies(t *testing.T) {
// TestReadPoliciesWhenThereAreNoPolicies calls model.ReadPolicies checking for a valid return when
// there are no policies to process
func TestReadPoliciesWhenThereAreNoPolicies(t *testing.T) {
mockConfig()
func (tg ReadFiles) TestReadPoliciesWhenThereAreNoPolicies(t *testing.T) {
path.Policies = func() ([]path.File, error) {
return []path.File{}, nil
}
@ -171,10 +170,9 @@ func TestReadPoliciesWhenThereAreNoPolicies(t *testing.T) {
// TestReadPoliciesFailsWhenInvalidPolicy calls model.ReadPolicies checking for an error return when
// there is an invalid policy
func TestReadPoliciesFailsWhenInvalidPolicy(t *testing.T) {
mockConfig()
func (tg ReadFiles) TestReadPoliciesFailsWhenInvalidPolicy(t *testing.T) {
path.Policies = func() ([]path.File, error) {
filePath := fmt.Sprintf("%s/../fixtures/policies/invalid-access.md", getRootPath())
filePath := fmt.Sprintf("%s/../fixtures/policies/invalid-access.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
return []path.File{
{FullPath: filePath, Info: fileInfo},
@ -188,9 +186,8 @@ func TestReadPoliciesFailsWhenInvalidPolicy(t *testing.T) {
}
// TestReadStandards calls model.ReadStandards checking for a valid return value.
func TestReadStandards(t *testing.T) {
mockConfig()
filePath := fmt.Sprintf("%s/standards/TSC-2017.yml", getRootPath())
func (tg ReadFiles) TestReadStandards(t *testing.T) {
filePath := fmt.Sprintf("%s/standards/TSC-2017.yml", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
path.Standards = func() ([]path.File, error) {
return []path.File{
@ -209,8 +206,7 @@ func TestReadStandards(t *testing.T) {
// TestReadStandardsWhenThereAreNoStandards calls model.ReadStandards checking for a valid return when
// there are no standards to process
func TestReadStandardsWhenThereAreNoStandards(t *testing.T) {
mockConfig()
func (tg ReadFiles) TestReadStandardsWhenThereAreNoStandards(t *testing.T) {
path.Standards = func() ([]path.File, error) {
return []path.File{}, nil
}
@ -226,10 +222,9 @@ func TestReadStandardsWhenThereAreNoStandards(t *testing.T) {
// TestReadStandardsFailsWhenInvalidStandard calls model.ReadStandards checking for an error return when
// there is an invalid standard
func TestReadStandardsFailsWhenInvalidStandard(t *testing.T) {
mockConfig()
func (tg ReadFiles) TestReadStandardsFailsWhenInvalidStandard(t *testing.T) {
path.Standards = func() ([]path.File, error) {
filePath := fmt.Sprintf("%s/../fixtures/standards/invalid-standard.yml", getRootPath())
filePath := fmt.Sprintf("%s/../fixtures/standards/invalid-standard.yml", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
return []path.File{
{FullPath: filePath, Info: fileInfo},
@ -241,18 +236,3 @@ func TestReadStandardsFailsWhenInvalidStandard(t *testing.T) {
t.Fatal(`ReadStandards() was expected to fail`, err)
}
}
func mockConfig() {
config.Config = func() *config.Project {
p := config.Project{}
cfgBytes, _ := ioutil.ReadFile(filepath.Join(getRootPath(), "comply.yml.example"))
yaml.Unmarshal(cfgBytes, &p)
return &p
}
}
func getRootPath() string {
_, fileName, _, _ := runtime.Caller(0)
fileDir := filepath.Dir(fileName)
return fmt.Sprintf("%s/../../example", fileDir)
}

View File

@ -29,11 +29,8 @@ func TestMarshal(t *testing.T) {
},
},
}
m, _ := json.Marshal(d)
encoded := string(m)
if !strings.Contains(encoded, "t1") ||
!strings.Contains(encoded, "a1") ||
!strings.Contains(encoded, "pro1") ||

View File

@ -135,9 +135,12 @@ func preprocessDoc(data *renderData, pol *model.Document, fullPath string) error
%% %s
---
header-includes: yes
head-content: "%s"
foot-content: "%s confidential %d"
header-includes: |
\usepackage{fancyhdr}
\pagestyle{fancy}
\fancyhead{}
\fancyhead[RO,RE]{%s}
\fancyfoot[LO,LE]{%s confidential %d}
---
%s

View File

@ -18,10 +18,7 @@ var pandocArgs = []string{"-f", "markdown+smart", "--toc", "-N", "--template", "
func pandoc(outputFilename string, errOutputCh chan error) {
if config.WhichPandoc() == config.UsePandoc {
err := pandocPandoc(outputFilename)
if err != nil {
errOutputCh <- err
}
pandocPandoc(outputFilename, errOutputCh)
} else {
dockerPandoc(outputFilename, errOutputCh)
}
@ -47,7 +44,7 @@ func dockerPandoc(outputFilename string, errOutputCh chan error) {
}
resp, err := cli.ContainerCreate(ctx, &container.Config{
Image: "strongdm/pandoc",
Image: "strongdm/pandoc:edge",
Cmd: pandocCmd},
hc, nil, nil, "")
@ -64,9 +61,11 @@ func dockerPandoc(outputFilename string, errOutputCh chan error) {
errOutputCh <- errors.Wrap(err, "unable to remove container")
return
}
errOutputCh <- nil
}()
if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{})
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to start Docker container")
return
}
@ -76,7 +75,7 @@ func dockerPandoc(outputFilename string, errOutputCh chan error) {
if resultValue.StatusCode != 0 {
err = <-chanErr
errOutputCh <-errors.Wrap(err, "error awaiting Docker container")
errOutputCh <- errors.Wrap(err, "error awaiting Docker container")
return
}
@ -93,12 +92,14 @@ func dockerPandoc(outputFilename string, errOutputCh chan error) {
}
// 🐼
func pandocPandoc(outputFilename string) error {
func pandocPandoc(outputFilename string, errOutputCh chan error) error {
cmd := exec.Command("pandoc", append(pandocArgs, fmt.Sprintf("output/%s", outputFilename), fmt.Sprintf("output/%s.md", outputFilename))...)
outputRaw, err := cmd.CombinedOutput()
if err != nil {
fmt.Println(string(outputRaw))
return errors.Wrap(err, "error calling pandoc")
errOutputCh <- errors.Wrap(err, "error calling pandoc")
} else {
errOutputCh <- nil
}
return nil
}

View File

@ -26,6 +26,12 @@ func pdf(output string, live bool, errCh chan error, wg *sync.WaitGroup) {
}
for _, policy := range policies {
renderToFilesystem(&pdfWG, errOutputCh, data, policy, live)
err = <-errOutputCh
if err != nil {
errCh <- err
wg.Done()
return
}
}
narratives, err := model.ReadNarratives()
@ -36,6 +42,12 @@ func pdf(output string, live bool, errCh chan error, wg *sync.WaitGroup) {
for _, narrative := range narratives {
renderToFilesystem(&pdfWG, errOutputCh, data, narrative, live)
err = <-errOutputCh
if err != nil {
errCh <- err
wg.Done()
return
}
}
pdfWG.Wait()

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,52 @@
package util
import (
"fmt"
"io/ioutil"
"path/filepath"
"reflect"
"runtime"
"testing"
"github.com/strongdm/comply/internal/config"
"gopkg.in/yaml.v2"
)
type TestFixture func()
func ExecuteTests(t *testing.T, testGroupType reflect.Type, beforeEach TestFixture, afterEach TestFixture) {
testGroup := reflect.New(testGroupType).Elem().Interface()
for i := 0; i < testGroupType.NumMethod(); i++ {
m := testGroupType.Method(i)
t.Run(m.Name, func(t *testing.T) {
if beforeEach != nil {
beforeEach()
}
in := []reflect.Value{reflect.ValueOf(testGroup), reflect.ValueOf(t)}
m.Func.Call(in)
if afterEach != nil {
afterEach()
}
})
}
}
func MockConfig() {
config.Config = func() *config.Project {
p := config.Project{}
cfgBytes, _ := ioutil.ReadFile(filepath.Join(GetRootPath(), "comply.yml.example"))
err := yaml.Unmarshal(cfgBytes, &p)
if err != nil {
return nil
}
return &p
}
}
func GetRootPath() string {
_, fileName, _, _ := runtime.Caller(0)
fileDir := filepath.Dir(fileName)
return fmt.Sprintf("%s/../../example", fileDir)
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

1
vendor/github.com/joho/godotenv/.gitignore generated vendored Normal file
View File

@ -0,0 +1 @@
.DS_Store

23
vendor/github.com/joho/godotenv/LICENCE generated vendored Normal file
View File

@ -0,0 +1,23 @@
Copyright (c) 2013 John Barton
MIT License
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

188
vendor/github.com/joho/godotenv/README.md generated vendored Normal file
View File

@ -0,0 +1,188 @@
# GoDotEnv ![CI](https://github.com/joho/godotenv/workflows/CI/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/joho/godotenv)](https://goreportcard.com/report/github.com/joho/godotenv)
A Go (golang) port of the Ruby dotenv project (which loads env vars from a .env file)
From the original Library:
> Storing configuration in the environment is one of the tenets of a twelve-factor app. Anything that is likely to change between deployment environmentssuch as resource handles for databases or credentials for external servicesshould be extracted from the code into environment variables.
>
> But it is not always practical to set environment variables on development machines or continuous integration servers where multiple projects are run. Dotenv load variables from a .env file into ENV when the environment is bootstrapped.
It can be used as a library (for loading in env for your own daemons etc) or as a bin command.
There is test coverage and CI for both linuxish and windows environments, but I make no guarantees about the bin version working on windows.
## Installation
As a library
```shell
go get github.com/joho/godotenv
```
or if you want to use it as a bin command
```shell
go get github.com/joho/godotenv/cmd/godotenv
```
## Usage
Add your application configuration to your `.env` file in the root of your project:
```shell
S3_BUCKET=YOURS3BUCKET
SECRET_KEY=YOURSECRETKEYGOESHERE
```
Then in your Go app you can do something like
```go
package main
import (
"github.com/joho/godotenv"
"log"
"os"
)
func main() {
err := godotenv.Load()
if err != nil {
log.Fatal("Error loading .env file")
}
s3Bucket := os.Getenv("S3_BUCKET")
secretKey := os.Getenv("SECRET_KEY")
// now do something with s3 or whatever
}
```
If you're even lazier than that, you can just take advantage of the autoload package which will read in `.env` on import
```go
import _ "github.com/joho/godotenv/autoload"
```
While `.env` in the project root is the default, you don't have to be constrained, both examples below are 100% legit
```go
_ = godotenv.Load("somerandomfile")
_ = godotenv.Load("filenumberone.env", "filenumbertwo.env")
```
If you want to be really fancy with your env file you can do comments and exports (below is a valid env file)
```shell
# I am a comment and that is OK
SOME_VAR=someval
FOO=BAR # comments at line end are OK too
export BAR=BAZ
```
Or finally you can do YAML(ish) style
```yaml
FOO: bar
BAR: baz
```
as a final aside, if you don't want godotenv munging your env you can just get a map back instead
```go
var myEnv map[string]string
myEnv, err := godotenv.Read()
s3Bucket := myEnv["S3_BUCKET"]
```
... or from an `io.Reader` instead of a local file
```go
reader := getRemoteFile()
myEnv, err := godotenv.Parse(reader)
```
... or from a `string` if you so desire
```go
content := getRemoteFileContent()
myEnv, err := godotenv.Unmarshal(content)
```
### Precedence & Conventions
Existing envs take precedence of envs that are loaded later.
The [convention](https://github.com/bkeepers/dotenv#what-other-env-files-can-i-use)
for managing multiple environments (i.e. development, test, production)
is to create an env named `{YOURAPP}_ENV` and load envs in this order:
```go
env := os.Getenv("FOO_ENV")
if "" == env {
env = "development"
}
godotenv.Load(".env." + env + ".local")
if "test" != env {
godotenv.Load(".env.local")
}
godotenv.Load(".env." + env)
godotenv.Load() // The Original .env
```
If you need to, you can also use `godotenv.Overload()` to defy this convention
and overwrite existing envs instead of only supplanting them. Use with caution.
### Command Mode
Assuming you've installed the command as above and you've got `$GOPATH/bin` in your `$PATH`
```
godotenv -f /some/path/to/.env some_command with some args
```
If you don't specify `-f` it will fall back on the default of loading `.env` in `PWD`
### Writing Env Files
Godotenv can also write a map representing the environment to a correctly-formatted and escaped file
```go
env, err := godotenv.Unmarshal("KEY=value")
err := godotenv.Write(env, "./.env")
```
... or to a string
```go
env, err := godotenv.Unmarshal("KEY=value")
content, err := godotenv.Marshal(env)
```
## Contributing
Contributions are most welcome! The parser itself is pretty stupidly naive and I wouldn't be surprised if it breaks with edge cases.
*code changes without tests will not be accepted*
1. Fork it
2. Create your feature branch (`git checkout -b my-new-feature`)
3. Commit your changes (`git commit -am 'Added some feature'`)
4. Push to the branch (`git push origin my-new-feature`)
5. Create new Pull Request
## Releases
Releases should follow [Semver](http://semver.org/) though the first couple of releases are `v1` and `v1.1`.
Use [annotated tags for all releases](https://github.com/joho/godotenv/issues/30). Example `git tag -a v1.2.1`
## CI
Linux: [![Build Status](https://travis-ci.org/joho/godotenv.svg?branch=master)](https://travis-ci.org/joho/godotenv) Windows: [![Build status](https://ci.appveyor.com/api/projects/status/9v40vnfvvgde64u4)](https://ci.appveyor.com/project/joho/godotenv)
## Who?
The original library [dotenv](https://github.com/bkeepers/dotenv) was written by [Brandon Keepers](http://opensoul.org/), and this port was done by [John Barton](https://johnbarton.co/) based off the tests/fixtures in the original library.

363
vendor/github.com/joho/godotenv/godotenv.go generated vendored Normal file
View File

@ -0,0 +1,363 @@
// Package godotenv is a go port of the ruby dotenv library (https://github.com/bkeepers/dotenv)
//
// Examples/readme can be found on the github page at https://github.com/joho/godotenv
//
// The TL;DR is that you make a .env file that looks something like
//
// SOME_ENV_VAR=somevalue
//
// and then in your go code you can call
//
// godotenv.Load()
//
// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR")
package godotenv
import (
"bufio"
"errors"
"fmt"
"io"
"os"
"os/exec"
"regexp"
"sort"
"strconv"
"strings"
)
const doubleQuoteSpecialChars = "\\\n\r\"!$`"
// Load will read your env file(s) and load them into ENV for this process.
//
// Call this function as close as possible to the start of your program (ideally in main)
//
// If you call Load without any args it will default to loading .env in the current path
//
// You can otherwise tell it which files to load (there can be more than one) like
//
// godotenv.Load("fileone", "filetwo")
//
// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults
func Load(filenames ...string) (err error) {
filenames = filenamesOrDefault(filenames)
for _, filename := range filenames {
err = loadFile(filename, false)
if err != nil {
return // return early on a spazout
}
}
return
}
// Overload will read your env file(s) and load them into ENV for this process.
//
// Call this function as close as possible to the start of your program (ideally in main)
//
// If you call Overload without any args it will default to loading .env in the current path
//
// You can otherwise tell it which files to load (there can be more than one) like
//
// godotenv.Overload("fileone", "filetwo")
//
// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars.
func Overload(filenames ...string) (err error) {
filenames = filenamesOrDefault(filenames)
for _, filename := range filenames {
err = loadFile(filename, true)
if err != nil {
return // return early on a spazout
}
}
return
}
// Read all env (with same file loading semantics as Load) but return values as
// a map rather than automatically writing values into env
func Read(filenames ...string) (envMap map[string]string, err error) {
filenames = filenamesOrDefault(filenames)
envMap = make(map[string]string)
for _, filename := range filenames {
individualEnvMap, individualErr := readFile(filename)
if individualErr != nil {
err = individualErr
return // return early on a spazout
}
for key, value := range individualEnvMap {
envMap[key] = value
}
}
return
}
// Parse reads an env file from io.Reader, returning a map of keys and values.
func Parse(r io.Reader) (envMap map[string]string, err error) {
envMap = make(map[string]string)
var lines []string
scanner := bufio.NewScanner(r)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
if err = scanner.Err(); err != nil {
return
}
for _, fullLine := range lines {
if !isIgnoredLine(fullLine) {
var key, value string
key, value, err = parseLine(fullLine, envMap)
if err != nil {
return
}
envMap[key] = value
}
}
return
}
//Unmarshal reads an env file from a string, returning a map of keys and values.
func Unmarshal(str string) (envMap map[string]string, err error) {
return Parse(strings.NewReader(str))
}
// Exec loads env vars from the specified filenames (empty map falls back to default)
// then executes the cmd specified.
//
// Simply hooks up os.Stdin/err/out to the command and calls Run()
//
// If you want more fine grained control over your command it's recommended
// that you use `Load()` or `Read()` and the `os/exec` package yourself.
func Exec(filenames []string, cmd string, cmdArgs []string) error {
Load(filenames...)
command := exec.Command(cmd, cmdArgs...)
command.Stdin = os.Stdin
command.Stdout = os.Stdout
command.Stderr = os.Stderr
return command.Run()
}
// Write serializes the given environment and writes it to a file
func Write(envMap map[string]string, filename string) error {
content, err := Marshal(envMap)
if err != nil {
return err
}
file, err := os.Create(filename)
if err != nil {
return err
}
defer file.Close()
_, err = file.WriteString(content + "\n")
if err != nil {
return err
}
file.Sync()
return err
}
// Marshal outputs the given environment as a dotenv-formatted environment file.
// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped.
func Marshal(envMap map[string]string) (string, error) {
lines := make([]string, 0, len(envMap))
for k, v := range envMap {
if d, err := strconv.Atoi(v); err == nil {
lines = append(lines, fmt.Sprintf(`%s=%d`, k, d))
} else {
lines = append(lines, fmt.Sprintf(`%s="%s"`, k, doubleQuoteEscape(v)))
}
}
sort.Strings(lines)
return strings.Join(lines, "\n"), nil
}
func filenamesOrDefault(filenames []string) []string {
if len(filenames) == 0 {
return []string{".env"}
}
return filenames
}
func loadFile(filename string, overload bool) error {
envMap, err := readFile(filename)
if err != nil {
return err
}
currentEnv := map[string]bool{}
rawEnv := os.Environ()
for _, rawEnvLine := range rawEnv {
key := strings.Split(rawEnvLine, "=")[0]
currentEnv[key] = true
}
for key, value := range envMap {
if !currentEnv[key] || overload {
os.Setenv(key, value)
}
}
return nil
}
func readFile(filename string) (envMap map[string]string, err error) {
file, err := os.Open(filename)
if err != nil {
return
}
defer file.Close()
return Parse(file)
}
var exportRegex = regexp.MustCompile(`^\s*(?:export\s+)?(.*?)\s*$`)
func parseLine(line string, envMap map[string]string) (key string, value string, err error) {
if len(line) == 0 {
err = errors.New("zero length string")
return
}
// ditch the comments (but keep quoted hashes)
if strings.Contains(line, "#") {
segmentsBetweenHashes := strings.Split(line, "#")
quotesAreOpen := false
var segmentsToKeep []string
for _, segment := range segmentsBetweenHashes {
if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 {
if quotesAreOpen {
quotesAreOpen = false
segmentsToKeep = append(segmentsToKeep, segment)
} else {
quotesAreOpen = true
}
}
if len(segmentsToKeep) == 0 || quotesAreOpen {
segmentsToKeep = append(segmentsToKeep, segment)
}
}
line = strings.Join(segmentsToKeep, "#")
}
firstEquals := strings.Index(line, "=")
firstColon := strings.Index(line, ":")
splitString := strings.SplitN(line, "=", 2)
if firstColon != -1 && (firstColon < firstEquals || firstEquals == -1) {
//this is a yaml-style line
splitString = strings.SplitN(line, ":", 2)
}
if len(splitString) != 2 {
err = errors.New("Can't separate key from value")
return
}
// Parse the key
key = splitString[0]
if strings.HasPrefix(key, "export") {
key = strings.TrimPrefix(key, "export")
}
key = strings.TrimSpace(key)
key = exportRegex.ReplaceAllString(splitString[0], "$1")
// Parse the value
value = parseValue(splitString[1], envMap)
return
}
var (
singleQuotesRegex = regexp.MustCompile(`\A'(.*)'\z`)
doubleQuotesRegex = regexp.MustCompile(`\A"(.*)"\z`)
escapeRegex = regexp.MustCompile(`\\.`)
unescapeCharsRegex = regexp.MustCompile(`\\([^$])`)
)
func parseValue(value string, envMap map[string]string) string {
// trim
value = strings.Trim(value, " ")
// check if we've got quoted values or possible escapes
if len(value) > 1 {
singleQuotes := singleQuotesRegex.FindStringSubmatch(value)
doubleQuotes := doubleQuotesRegex.FindStringSubmatch(value)
if singleQuotes != nil || doubleQuotes != nil {
// pull the quotes off the edges
value = value[1 : len(value)-1]
}
if doubleQuotes != nil {
// expand newlines
value = escapeRegex.ReplaceAllStringFunc(value, func(match string) string {
c := strings.TrimPrefix(match, `\`)
switch c {
case "n":
return "\n"
case "r":
return "\r"
default:
return match
}
})
// unescape characters
value = unescapeCharsRegex.ReplaceAllString(value, "$1")
}
if singleQuotes == nil {
value = expandVariables(value, envMap)
}
}
return value
}
var expandVarRegex = regexp.MustCompile(`(\\)?(\$)(\()?\{?([A-Z0-9_]+)?\}?`)
func expandVariables(v string, m map[string]string) string {
return expandVarRegex.ReplaceAllStringFunc(v, func(s string) string {
submatch := expandVarRegex.FindStringSubmatch(s)
if submatch == nil {
return s
}
if submatch[1] == "\\" || submatch[2] == "(" {
return submatch[0][1:]
} else if submatch[4] != "" {
return m[submatch[4]]
}
return s
})
}
func isIgnoredLine(line string) bool {
trimmedLine := strings.TrimSpace(line)
return len(trimmedLine) == 0 || strings.HasPrefix(trimmedLine, "#")
}
func doubleQuoteEscape(line string) string {
for _, c := range doubleQuoteSpecialChars {
toReplace := "\\" + string(c)
if c == '\n' {
toReplace = `\n`
}
if c == '\r' {
toReplace = `\r`
}
line = strings.Replace(line, string(c), toReplace, -1)
}
return line
}

5
vendor/github.com/joho/godotenv/renovate.json generated vendored Normal file
View File

@ -0,0 +1,5 @@
{
"extends": [
"config:base"
]
}

3
vendor/modules.txt vendored
View File

@ -116,6 +116,9 @@ github.com/hashicorp/go-retryablehttp
# github.com/jcelliott/lumber v0.0.0-20160324203708-dd349441af25
## explicit
github.com/jcelliott/lumber
# github.com/joho/godotenv v1.4.0
## explicit; go 1.12
github.com/joho/godotenv
# github.com/juju/ansiterm v0.0.0-20210929141451-8b71cc96ebdc
## explicit; go 1.14
github.com/juju/ansiterm