From 49e950c3c029fed8c05c29cfd6dd190122505be7 Mon Sep 17 00:00:00 2001 From: Justin McCarthy Date: Wed, 23 May 2018 16:48:35 -0700 Subject: [PATCH] If `pandoc` appears in the path, it will be preferred over Docker. The pandoc version must be 2.2.1 or greater. Defaults can be overridden by an optional "pandoc: pandoc" or "pandoc: docker" in the comply.yml. --- internal/cli/app.go | 27 ++++++---- internal/config/config.go | 31 +++++++++++ internal/render/narrative.go | 64 +--------------------- internal/render/pandoc.go | 100 +++++++++++++++++++++++++++++++++++ internal/render/policy.go | 62 +--------------------- 5 files changed, 151 insertions(+), 133 deletions(-) create mode 100644 internal/render/pandoc.go diff --git a/internal/cli/app.go b/internal/cli/app.go index a4f3fd3..42b0384 100644 --- a/internal/cli/app.go +++ b/internal/cli/app.go @@ -100,16 +100,16 @@ func ticketingMustBeConfigured(c *cli.Context) error { } func pandocMustExist(c *cli.Context) error { - pandocErr := fmt.Errorf("Please install either Docker or the pandoc package and re-run `%s`", c.Command.Name) + eitherMustExistErr := fmt.Errorf("Please install either Docker or the pandoc package and re-run `%s`", c.Command.Name) - err := pandocBinaryMustExist(c) - fmt.Println(err) - if err != nil { - err = dockerMustExist(c) - if err != nil { - return pandocErr - } + pandocExistErr := pandocBinaryMustExist(c) + dockerExistErr := dockerMustExist(c) + config.SetPandoc(pandocExistErr == nil, dockerExistErr == nil) + + if pandocExistErr != nil && dockerExistErr != nil { + return eitherMustExistErr } + return nil } @@ -197,12 +197,17 @@ func dockerMustExist(c *cli.Context) error { } func cleanContainers(c *cli.Context) error { - dockerErr := fmt.Errorf("Docker must be available in order to run `%s`", c.Command.Name) - ctx := context.Background() cli, err := client.NewEnvClient() if err != nil { - return dockerErr + // no Docker? nothing to clean. + return nil + } + + _, err = cli.Ping(ctx) + if err != nil { + // no Docker? nothing to clean. + return nil } containers, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true}) diff --git a/internal/config/config.go b/internal/config/config.go index e0c0f09..cdac061 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,6 +10,15 @@ import ( var projectRoot string +var dockerAvailable, pandocAvailable bool + +const ( + // UseDocker invokes pandoc within Docker + UseDocker = "docker" + // UsePandoc invokes pandoc directly + UsePandoc = "pandoc" +) + // SetProjectRoot is used by the test suite. func SetProjectRoot(dir string) { projectRoot = dir @@ -17,10 +26,32 @@ func SetProjectRoot(dir string) { type Project struct { Name string `yaml:"name"` + Pandoc string `yaml:"pandoc,omitempty"` FilePrefix string `yaml:"filePrefix"` Tickets map[string]interface{} `yaml:"tickets"` } +// SetPandoc records pandoc availability during initialization +func SetPandoc(pandoc bool, docker bool) { + pandocAvailable = pandoc + dockerAvailable = docker +} + +// WhichPandoc indicates which pandoc invocation path should be used +func WhichPandoc() string { + cfg := Config() + if cfg.Pandoc == UsePandoc { + return UsePandoc + } + if cfg.Pandoc == UseDocker { + return UseDocker + } + if pandocAvailable { + return UsePandoc + } + return UseDocker +} + // YAML is the parsed contents of ProjectRoot()/config.yml. func YAML() map[interface{}]interface{} { m := make(map[interface{}]interface{}) diff --git a/internal/render/narrative.go b/internal/render/narrative.go index f72bd90..3aeaff7 100644 --- a/internal/render/narrative.go +++ b/internal/render/narrative.go @@ -2,7 +2,6 @@ package render import ( "bytes" - "context" "fmt" "io/ioutil" "os" @@ -12,9 +11,6 @@ import ( "text/template" "time" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" "github.com/pkg/errors" "github.com/strongdm/comply/internal/config" "github.com/strongdm/comply/internal/model" @@ -28,75 +24,19 @@ func renderNarrativeToDisk(wg *sync.WaitGroup, errOutputCh chan error, data *ren } recordModified(narrative.FullPath, narrative.ModifiedAt) - ctx := context.Background() - cli, err := client.NewEnvClient() - if err != nil { - errOutputCh <- errors.Wrap(err, "unable to read Docker environment") - return - } - - pwd, err := os.Getwd() - if err != nil { - errOutputCh <- errors.Wrap(err, "unable to get workding directory") - return - } - - hc := &container.HostConfig{ - Binds: []string{pwd + ":/source"}, - } - wg.Add(1) go func(p *model.Narrative) { defer wg.Done() outputFilename := p.OutputFilename // save preprocessed markdown - err = preprocessNarrative(data, p, filepath.Join(".", "output", outputFilename+".md")) + err := preprocessNarrative(data, p, filepath.Join(".", "output", outputFilename+".md")) if err != nil { errOutputCh <- errors.Wrap(err, "unable to preprocess") return } - cmd := []string{"--smart", "--toc", "-N", "--template=/source/templates/default.latex", "-o", - fmt.Sprintf("/source/output/%s", outputFilename), - fmt.Sprintf("/source/output/%s.md", outputFilename)} - - resp, err := cli.ContainerCreate(ctx, &container.Config{ - Image: "strongdm/pandoc", - Cmd: cmd}, - hc, nil, "") - - if err != nil { - errOutputCh <- errors.Wrap(err, "unable to create Docker container") - return - } - - defer func() { - timeout := 2 * time.Second - cli.ContainerStop(ctx, resp.ID, &timeout) - err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true}) - if err != nil { - errOutputCh <- errors.Wrap(err, "unable to remove container") - return - } - }() - - if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { - errOutputCh <- errors.Wrap(err, "unable to start Docker container") - return - } - - _, err = cli.ContainerWait(ctx, resp.ID) - if err != nil { - errOutputCh <- errors.Wrap(err, "error awaiting Docker container") - return - } - - _, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true}) - if err != nil { - errOutputCh <- errors.Wrap(err, "error reading Docker container logs") - return - } + pandoc(outputFilename, errOutputCh) // remove preprocessed markdown err = os.Remove(filepath.Join(".", "output", outputFilename+".md")) diff --git a/internal/render/pandoc.go b/internal/render/pandoc.go new file mode 100644 index 0000000..2880feb --- /dev/null +++ b/internal/render/pandoc.go @@ -0,0 +1,100 @@ +package render + +import ( + "context" + "fmt" + "os" + "os/exec" + "time" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/container" + "github.com/docker/docker/client" + "github.com/pkg/errors" + "github.com/strongdm/comply/internal/config" +) + +func pandoc(outputFilename string, errOutputCh chan error) { + if config.WhichPandoc() == config.UsePandoc { + err := pandocPandoc(outputFilename) + if err != nil { + errOutputCh <- err + } + } else { + dockerPandoc(outputFilename, errOutputCh) + } +} + +func dockerPandoc(outputFilename string, errOutputCh chan error) { + // TODO: switch to new args once docker image is updated + // cmd21 := []string{"-f", "markdown+smart", "--toc", "-N", "--template", "templates/default.latex", "-o", fmt.Sprintf("output/%s", outputFilename), fmt.Sprintf("output/%s.md", outputFilename)} + cmd19 := []string{"--smart", "--toc", "-N", "--template=/source/templates/default.latex", "-o", + fmt.Sprintf("/source/output/%s", outputFilename), + fmt.Sprintf("/source/output/%s.md", outputFilename)} + + ctx := context.Background() + cli, err := client.NewEnvClient() + if err != nil { + errOutputCh <- errors.Wrap(err, "unable to read Docker environment") + return + } + + pwd, err := os.Getwd() + if err != nil { + errOutputCh <- errors.Wrap(err, "unable to get workding directory") + return + } + + hc := &container.HostConfig{ + Binds: []string{pwd + ":/source"}, + } + + resp, err := cli.ContainerCreate(ctx, &container.Config{ + Image: "strongdm/pandoc", + Cmd: cmd19}, + hc, nil, "") + + if err != nil { + errOutputCh <- errors.Wrap(err, "unable to create Docker container") + return + } + + defer func() { + timeout := 2 * time.Second + cli.ContainerStop(ctx, resp.ID, &timeout) + err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true}) + if err != nil { + errOutputCh <- errors.Wrap(err, "unable to remove container") + return + } + }() + + if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { + errOutputCh <- errors.Wrap(err, "unable to start Docker container") + return + } + + _, err = cli.ContainerWait(ctx, resp.ID) + if err != nil { + errOutputCh <- errors.Wrap(err, "error awaiting Docker container") + return + } + + _, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true}) + if err != nil { + errOutputCh <- errors.Wrap(err, "error reading Docker container logs") + return + } +} + +// 🐼 +func pandocPandoc(outputFilename string) error { + // -f markdown+smart --toc -N --template=templates/default.latex -o output/%s output/%s.md + cmd := exec.Command("pandoc", "-f", "markdown+smart", "--toc", "-N", "--template", "templates/default.latex", "-o", fmt.Sprintf("output/%s", outputFilename), fmt.Sprintf("output/%s.md", outputFilename)) + outputRaw, err := cmd.CombinedOutput() + if err != nil { + fmt.Println(string(outputRaw)) + return errors.Wrap(err, "error calling pandoc") + } + return nil +} diff --git a/internal/render/policy.go b/internal/render/policy.go index ea3a411..afabadb 100644 --- a/internal/render/policy.go +++ b/internal/render/policy.go @@ -2,7 +2,6 @@ package render import ( "bytes" - "context" "fmt" "io/ioutil" "os" @@ -12,9 +11,6 @@ import ( "text/template" "time" - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/container" - "github.com/docker/docker/client" "github.com/pkg/errors" "github.com/strongdm/comply/internal/config" "github.com/strongdm/comply/internal/model" @@ -28,73 +24,19 @@ func renderPolicyToDisk(wg *sync.WaitGroup, errOutputCh chan error, data *render } recordModified(policy.FullPath, policy.ModifiedAt) - ctx := context.Background() - cli, err := client.NewEnvClient() - if err != nil { - errOutputCh <- errors.Wrap(err, "unable to read Docker environment") - return - } - - pwd, err := os.Getwd() - if err != nil { - errOutputCh <- errors.Wrap(err, "unable to get workding directory") - return - } - - hc := &container.HostConfig{ - Binds: []string{pwd + ":/source"}, - } - wg.Add(1) go func(p *model.Policy) { defer wg.Done() outputFilename := p.OutputFilename // save preprocessed markdown - err = preprocessPolicy(data, p, filepath.Join(".", "output", outputFilename+".md")) + err := preprocessPolicy(data, p, filepath.Join(".", "output", outputFilename+".md")) if err != nil { errOutputCh <- errors.Wrap(err, "unable to preprocess") return } - resp, err := cli.ContainerCreate(ctx, &container.Config{ - Image: "strongdm/pandoc", - Cmd: []string{"--smart", "--toc", "-N", "--template=/source/templates/default.latex", "-o", - fmt.Sprintf("/source/output/%s", outputFilename), - fmt.Sprintf("/source/output/%s.md", outputFilename), - }, - }, hc, nil, "") - if err != nil { - errOutputCh <- errors.Wrap(err, "unable to create Docker container") - return - } - - defer func() { - timeout := 2 * time.Second - cli.ContainerStop(ctx, resp.ID, &timeout) - err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true}) - if err != nil { - errOutputCh <- errors.Wrap(err, "unable to remove container") - return - } - }() - - if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil { - errOutputCh <- errors.Wrap(err, "unable to start Docker container") - return - } - - _, err = cli.ContainerWait(ctx, resp.ID) - if err != nil { - errOutputCh <- errors.Wrap(err, "error awaiting Docker container") - return - } - - _, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true}) - if err != nil { - errOutputCh <- errors.Wrap(err, "error reading Docker container logs") - return - } + pandoc(outputFilename, errOutputCh) // remove preprocessed markdown err = os.Remove(filepath.Join(".", "output", outputFilename+".md"))