1
0
mirror of https://github.com/strongdm/comply synced 2024-11-22 07:34:54 +00:00

If pandoc appears in the path, it will be preferred over Docker.

The pandoc version must be 2.2.1 or greater.

Defaults can be overridden by an optional "pandoc: pandoc"
or "pandoc: docker" in the comply.yml.
This commit is contained in:
Justin McCarthy 2018-05-23 16:48:35 -07:00
parent ff350a2b89
commit 49e950c3c0
No known key found for this signature in database
GPG Key ID: 900437410E142A48
5 changed files with 151 additions and 133 deletions

View File

@ -100,16 +100,16 @@ func ticketingMustBeConfigured(c *cli.Context) error {
} }
func pandocMustExist(c *cli.Context) error { func pandocMustExist(c *cli.Context) error {
pandocErr := fmt.Errorf("Please install either Docker or the pandoc package and re-run `%s`", c.Command.Name) eitherMustExistErr := fmt.Errorf("Please install either Docker or the pandoc package and re-run `%s`", c.Command.Name)
err := pandocBinaryMustExist(c) pandocExistErr := pandocBinaryMustExist(c)
fmt.Println(err) dockerExistErr := dockerMustExist(c)
if err != nil { config.SetPandoc(pandocExistErr == nil, dockerExistErr == nil)
err = dockerMustExist(c)
if err != nil { if pandocExistErr != nil && dockerExistErr != nil {
return pandocErr return eitherMustExistErr
}
} }
return nil return nil
} }
@ -197,12 +197,17 @@ func dockerMustExist(c *cli.Context) error {
} }
func cleanContainers(c *cli.Context) error { func cleanContainers(c *cli.Context) error {
dockerErr := fmt.Errorf("Docker must be available in order to run `%s`", c.Command.Name)
ctx := context.Background() ctx := context.Background()
cli, err := client.NewEnvClient() cli, err := client.NewEnvClient()
if err != nil { if err != nil {
return dockerErr // no Docker? nothing to clean.
return nil
}
_, err = cli.Ping(ctx)
if err != nil {
// no Docker? nothing to clean.
return nil
} }
containers, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true}) containers, err := cli.ContainerList(ctx, types.ContainerListOptions{All: true})

View File

@ -10,6 +10,15 @@ import (
var projectRoot string var projectRoot string
var dockerAvailable, pandocAvailable bool
const (
// UseDocker invokes pandoc within Docker
UseDocker = "docker"
// UsePandoc invokes pandoc directly
UsePandoc = "pandoc"
)
// SetProjectRoot is used by the test suite. // SetProjectRoot is used by the test suite.
func SetProjectRoot(dir string) { func SetProjectRoot(dir string) {
projectRoot = dir projectRoot = dir
@ -17,10 +26,32 @@ func SetProjectRoot(dir string) {
type Project struct { type Project struct {
Name string `yaml:"name"` Name string `yaml:"name"`
Pandoc string `yaml:"pandoc,omitempty"`
FilePrefix string `yaml:"filePrefix"` FilePrefix string `yaml:"filePrefix"`
Tickets map[string]interface{} `yaml:"tickets"` Tickets map[string]interface{} `yaml:"tickets"`
} }
// SetPandoc records pandoc availability during initialization
func SetPandoc(pandoc bool, docker bool) {
pandocAvailable = pandoc
dockerAvailable = docker
}
// WhichPandoc indicates which pandoc invocation path should be used
func WhichPandoc() string {
cfg := Config()
if cfg.Pandoc == UsePandoc {
return UsePandoc
}
if cfg.Pandoc == UseDocker {
return UseDocker
}
if pandocAvailable {
return UsePandoc
}
return UseDocker
}
// YAML is the parsed contents of ProjectRoot()/config.yml. // YAML is the parsed contents of ProjectRoot()/config.yml.
func YAML() map[interface{}]interface{} { func YAML() map[interface{}]interface{} {
m := make(map[interface{}]interface{}) m := make(map[interface{}]interface{})

View File

@ -2,7 +2,6 @@ package render
import ( import (
"bytes" "bytes"
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
@ -12,9 +11,6 @@ import (
"text/template" "text/template"
"time" "time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/strongdm/comply/internal/config" "github.com/strongdm/comply/internal/config"
"github.com/strongdm/comply/internal/model" "github.com/strongdm/comply/internal/model"
@ -28,75 +24,19 @@ func renderNarrativeToDisk(wg *sync.WaitGroup, errOutputCh chan error, data *ren
} }
recordModified(narrative.FullPath, narrative.ModifiedAt) recordModified(narrative.FullPath, narrative.ModifiedAt)
ctx := context.Background()
cli, err := client.NewEnvClient()
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to read Docker environment")
return
}
pwd, err := os.Getwd()
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to get workding directory")
return
}
hc := &container.HostConfig{
Binds: []string{pwd + ":/source"},
}
wg.Add(1) wg.Add(1)
go func(p *model.Narrative) { go func(p *model.Narrative) {
defer wg.Done() defer wg.Done()
outputFilename := p.OutputFilename outputFilename := p.OutputFilename
// save preprocessed markdown // save preprocessed markdown
err = preprocessNarrative(data, p, filepath.Join(".", "output", outputFilename+".md")) err := preprocessNarrative(data, p, filepath.Join(".", "output", outputFilename+".md"))
if err != nil { if err != nil {
errOutputCh <- errors.Wrap(err, "unable to preprocess") errOutputCh <- errors.Wrap(err, "unable to preprocess")
return return
} }
cmd := []string{"--smart", "--toc", "-N", "--template=/source/templates/default.latex", "-o", pandoc(outputFilename, errOutputCh)
fmt.Sprintf("/source/output/%s", outputFilename),
fmt.Sprintf("/source/output/%s.md", outputFilename)}
resp, err := cli.ContainerCreate(ctx, &container.Config{
Image: "strongdm/pandoc",
Cmd: cmd},
hc, nil, "")
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to create Docker container")
return
}
defer func() {
timeout := 2 * time.Second
cli.ContainerStop(ctx, resp.ID, &timeout)
err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to remove container")
return
}
}()
if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
errOutputCh <- errors.Wrap(err, "unable to start Docker container")
return
}
_, err = cli.ContainerWait(ctx, resp.ID)
if err != nil {
errOutputCh <- errors.Wrap(err, "error awaiting Docker container")
return
}
_, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true})
if err != nil {
errOutputCh <- errors.Wrap(err, "error reading Docker container logs")
return
}
// remove preprocessed markdown // remove preprocessed markdown
err = os.Remove(filepath.Join(".", "output", outputFilename+".md")) err = os.Remove(filepath.Join(".", "output", outputFilename+".md"))

100
internal/render/pandoc.go Normal file
View File

@ -0,0 +1,100 @@
package render
import (
"context"
"fmt"
"os"
"os/exec"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
"github.com/pkg/errors"
"github.com/strongdm/comply/internal/config"
)
func pandoc(outputFilename string, errOutputCh chan error) {
if config.WhichPandoc() == config.UsePandoc {
err := pandocPandoc(outputFilename)
if err != nil {
errOutputCh <- err
}
} else {
dockerPandoc(outputFilename, errOutputCh)
}
}
func dockerPandoc(outputFilename string, errOutputCh chan error) {
// TODO: switch to new args once docker image is updated
// cmd21 := []string{"-f", "markdown+smart", "--toc", "-N", "--template", "templates/default.latex", "-o", fmt.Sprintf("output/%s", outputFilename), fmt.Sprintf("output/%s.md", outputFilename)}
cmd19 := []string{"--smart", "--toc", "-N", "--template=/source/templates/default.latex", "-o",
fmt.Sprintf("/source/output/%s", outputFilename),
fmt.Sprintf("/source/output/%s.md", outputFilename)}
ctx := context.Background()
cli, err := client.NewEnvClient()
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to read Docker environment")
return
}
pwd, err := os.Getwd()
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to get workding directory")
return
}
hc := &container.HostConfig{
Binds: []string{pwd + ":/source"},
}
resp, err := cli.ContainerCreate(ctx, &container.Config{
Image: "strongdm/pandoc",
Cmd: cmd19},
hc, nil, "")
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to create Docker container")
return
}
defer func() {
timeout := 2 * time.Second
cli.ContainerStop(ctx, resp.ID, &timeout)
err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to remove container")
return
}
}()
if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
errOutputCh <- errors.Wrap(err, "unable to start Docker container")
return
}
_, err = cli.ContainerWait(ctx, resp.ID)
if err != nil {
errOutputCh <- errors.Wrap(err, "error awaiting Docker container")
return
}
_, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true})
if err != nil {
errOutputCh <- errors.Wrap(err, "error reading Docker container logs")
return
}
}
// 🐼
func pandocPandoc(outputFilename string) error {
// -f markdown+smart --toc -N --template=templates/default.latex -o output/%s output/%s.md
cmd := exec.Command("pandoc", "-f", "markdown+smart", "--toc", "-N", "--template", "templates/default.latex", "-o", fmt.Sprintf("output/%s", outputFilename), fmt.Sprintf("output/%s.md", outputFilename))
outputRaw, err := cmd.CombinedOutput()
if err != nil {
fmt.Println(string(outputRaw))
return errors.Wrap(err, "error calling pandoc")
}
return nil
}

View File

@ -2,7 +2,6 @@ package render
import ( import (
"bytes" "bytes"
"context"
"fmt" "fmt"
"io/ioutil" "io/ioutil"
"os" "os"
@ -12,9 +11,6 @@ import (
"text/template" "text/template"
"time" "time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/strongdm/comply/internal/config" "github.com/strongdm/comply/internal/config"
"github.com/strongdm/comply/internal/model" "github.com/strongdm/comply/internal/model"
@ -28,73 +24,19 @@ func renderPolicyToDisk(wg *sync.WaitGroup, errOutputCh chan error, data *render
} }
recordModified(policy.FullPath, policy.ModifiedAt) recordModified(policy.FullPath, policy.ModifiedAt)
ctx := context.Background()
cli, err := client.NewEnvClient()
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to read Docker environment")
return
}
pwd, err := os.Getwd()
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to get workding directory")
return
}
hc := &container.HostConfig{
Binds: []string{pwd + ":/source"},
}
wg.Add(1) wg.Add(1)
go func(p *model.Policy) { go func(p *model.Policy) {
defer wg.Done() defer wg.Done()
outputFilename := p.OutputFilename outputFilename := p.OutputFilename
// save preprocessed markdown // save preprocessed markdown
err = preprocessPolicy(data, p, filepath.Join(".", "output", outputFilename+".md")) err := preprocessPolicy(data, p, filepath.Join(".", "output", outputFilename+".md"))
if err != nil { if err != nil {
errOutputCh <- errors.Wrap(err, "unable to preprocess") errOutputCh <- errors.Wrap(err, "unable to preprocess")
return return
} }
resp, err := cli.ContainerCreate(ctx, &container.Config{ pandoc(outputFilename, errOutputCh)
Image: "strongdm/pandoc",
Cmd: []string{"--smart", "--toc", "-N", "--template=/source/templates/default.latex", "-o",
fmt.Sprintf("/source/output/%s", outputFilename),
fmt.Sprintf("/source/output/%s.md", outputFilename),
},
}, hc, nil, "")
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to create Docker container")
return
}
defer func() {
timeout := 2 * time.Second
cli.ContainerStop(ctx, resp.ID, &timeout)
err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})
if err != nil {
errOutputCh <- errors.Wrap(err, "unable to remove container")
return
}
}()
if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
errOutputCh <- errors.Wrap(err, "unable to start Docker container")
return
}
_, err = cli.ContainerWait(ctx, resp.ID)
if err != nil {
errOutputCh <- errors.Wrap(err, "error awaiting Docker container")
return
}
_, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true})
if err != nil {
errOutputCh <- errors.Wrap(err, "error reading Docker container logs")
return
}
// remove preprocessed markdown // remove preprocessed markdown
err = os.Remove(filepath.Join(".", "output", outputFilename+".md")) err = os.Remove(filepath.Join(".", "output", outputFilename+".md"))