1
0
mirror of https://github.com/strongdm/comply synced 2025-12-06 06:14:09 +00:00

Update pandoc dockerfile and latex template and fix infinite build, fix pandoc always pull and fix strikethrough (#100, #103 and #115)

This commit is contained in:
wallrony
2021-10-19 16:05:25 -03:00
parent fd4aab67be
commit f4f46bad4d
30 changed files with 2847 additions and 508 deletions

View File

@@ -137,10 +137,10 @@ func notifyVersion(c *cli.Context) error {
func pandocMustExist(c *cli.Context) error {
eitherMustExistErr := fmt.Errorf("\n\nPlease install either Docker or the pandoc package and re-run `%s`. Find OS-specific pandoc installation instructions at: https://pandoc.org/installing.html", c.Command.Name)
pandocExistErr, found, goodVersion, pdfLatex := pandocBinaryMustExist(c)
pandocBinaryExistErr, found, goodVersion, pdfLatex := pandocBinaryMustExist(c)
dockerExistErr, inPath, isRunning := dockerMustExist(c)
config.SetPandoc(pandocExistErr == nil, dockerExistErr == nil)
config.SetPandoc(pandocBinaryExistErr == nil, dockerExistErr == nil)
check := func(b bool) string {
if b {
return "✔"
@@ -150,7 +150,7 @@ func pandocMustExist(c *cli.Context) error {
}
if pandocExistErr != nil && dockerExistErr != nil {
if pandocBinaryExistErr != nil && dockerExistErr != nil {
fmt.Printf(`
[%s] pandoc binary installed and in PATH
@@ -165,14 +165,20 @@ func pandocMustExist(c *cli.Context) error {
}
// if we don't have pandoc, but we do have docker, execute a pull
if (pandocExistErr != nil && dockerExistErr == nil) || config.WhichPandoc() == config.UseDocker {
dockerPull(c)
if !pandocImageExists(context.Background()) && ((pandocBinaryExistErr != nil && dockerExistErr == nil) || config.WhichPandoc() == config.UseDocker) {
canPullPandoc := strings.TrimSpace(strings.ToLower(os.Getenv("COMPLY_USE_LOCAL_PANDOC"))) != "true"
if canPullPandoc {
fmt.Println("Pulling docker image")
dockerPull(c)
} else {
return fmt.Errorf("Local Pandoc not found. Please set COMPLY_USE_LOCAL_PANDOC to false")
}
}
return nil
}
func pandocBinaryMustExist(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
var pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
cmd := exec.Command("pandoc", "-v")
outputRaw, err := cmd.Output()
@@ -230,7 +236,7 @@ func pandocBinaryMustExist(c *cli.Context) (e error, found, goodVersion, pdfLate
return e, found, goodVersion, pdfLatex
}
func dockerMustExist(c *cli.Context) (e error, inPath, isRunning bool) {
var dockerMustExist = func(c *cli.Context) (e error, inPath, isRunning bool) {
dockerErr := fmt.Errorf("Docker must be available in order to run `%s`", c.Command.Name)
inPath = true
@@ -257,7 +263,26 @@ func dockerMustExist(c *cli.Context) (e error, inPath, isRunning bool) {
return nil, inPath, isRunning
}
func dockerPull(c *cli.Context) error {
var pandocImageExists = func(ctx context.Context) bool {
cli, err := client.NewEnvClient()
if err != nil {
return false
}
options := types.ImageListOptions{All: true}
imageList, err := cli.ImageList(ctx, options)
if err != nil {
return false
}
for _, image := range imageList {
if strings.Contains(image.RepoTags[0], "strongdm/pandoc") {
return true
}
}
return false
}
var dockerPull = func(c *cli.Context) error {
dockerErr := fmt.Errorf("Docker must be available in order to run `%s`", c.Command.Name)
ctx := context.Background()
@@ -275,7 +300,7 @@ func dockerPull(c *cli.Context) error {
select {
case <-longishPull:
fmt.Print("Pulling strongdm/pandoc:latest Docker image (this will take some time) ")
fmt.Print("Pulling strongdm/pandoc:edge Docker image (this will take some time) ")
go func() {
for {
@@ -294,7 +319,7 @@ func dockerPull(c *cli.Context) error {
}
}()
r, err := cli.ImagePull(ctx, "strongdm/pandoc:latest", types.ImagePullOptions{})
r, err := cli.ImagePull(ctx, "strongdm/pandoc:edge", types.ImagePullOptions{})
if err != nil {
return dockerErr
}

189
internal/cli/app_test.go Normal file
View File

@@ -0,0 +1,189 @@
package cli
import (
"context"
"errors"
"os"
"reflect"
"testing"
"github.com/strongdm/comply/internal/util"
"github.com/urfave/cli"
)
type PandocMustExist struct{}
func TestPandocMustExist(t *testing.T) {
util.ExecuteTests(t, reflect.TypeOf(PandocMustExist{}), beforeEach, nil)
}
func beforeEach() {
util.MockConfig()
}
func (tg PandocMustExist) WhenBinaryExists(t *testing.T) {
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return nil, true, true, true
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return errors.New("docker doesn't exist"), false, false
}
pandocImageExists = func(ctx context.Context) bool {
return false
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
}
func (tg PandocMustExist) WhenImageExists(t *testing.T) {
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, true, true
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return errors.New("docker doesn't exist"), false, false
}
pandocImageExists = func(ctx context.Context) bool {
return true
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
}
func (tg PandocMustExist) WhenBinaryAndImageDontExists(t *testing.T) {
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, false, false
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return nil, true, false
}
pandocImageExists = func(ctx context.Context) bool {
return false
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != true {
t.Fatal("Docker wasn't pulled")
}
}
func (tg PandocMustExist) WhenCannotPullPandoc(t *testing.T) {
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, false, false
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return errors.New("docker doesn't exist"), false, false
}
pandocImageExists = func(ctx context.Context) bool {
return false
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
}
func (tg PandocMustExist) WhenMustUseLocalPandoc(t *testing.T) {
os.Setenv("COMPLY_USE_LOCAL_PANDOC", "true")
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, false, false
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return errors.New("docker doesn't exist"), false, false
}
pandocImageExists = func(ctx context.Context) bool {
return true
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
os.Clearenv()
}
func (tg PandocMustExist) WhenPandocDontExistsAndCannotPull(t *testing.T) {
os.Setenv("COMPLY_USE_LOCAL_PANDOC", "true")
dockerPullCalled := false
pandocBinaryMustExist = func(c *cli.Context) (e error, found, goodVersion, pdfLatex bool) {
return errors.New("binary doesn't exist"), false, false, false
}
dockerMustExist = func(c *cli.Context) (e error, inPath bool, isRunning bool) {
return nil, true, false
}
pandocImageExists = func(ctx context.Context) bool {
return false
}
dockerPull = func(c *cli.Context) error {
dockerPullCalled = true
return nil
}
pandocMustExist(&cli.Context{})
if dockerPullCalled != false {
t.Fatal("Docker was pulled")
}
os.Clearenv()
}

View File

@@ -82,7 +82,7 @@ func Exists() bool {
}
// Config is the parsed contents of ProjectRoot()/config.yml.
func Config() *Project {
var Config = func() *Project {
p := Project{}
cfgBytes, err := ioutil.ReadFile(filepath.Join(ProjectRoot(), "comply.yml"))
if err != nil {

View File

@@ -84,6 +84,10 @@ func ReadStandards() ([]*Standard, error) {
}
yaml.Unmarshal(sBytes, &s)
err = yaml.Unmarshal(sBytes, &s)
if err != nil {
return nil, errors.Wrap(err, "unable to parse "+f.FullPath)
}
standards = append(standards, s)
}
@@ -101,7 +105,10 @@ func ReadNarratives() ([]*Document, error) {
for _, f := range files {
n := &Document{}
mdmd := loadMDMD(f.FullPath)
mdmd, err := loadMDMD(f.FullPath)
if err != nil {
return nil, err
}
err = yaml.Unmarshal([]byte(mdmd.yaml), &n)
if err != nil {
return nil, errors.Wrap(err, "unable to parse "+f.FullPath)
@@ -126,7 +133,10 @@ func ReadProcedures() ([]*Procedure, error) {
for _, f := range files {
p := &Procedure{}
mdmd := loadMDMD(f.FullPath)
mdmd, err := loadMDMD(f.FullPath)
if err != nil {
return nil, err
}
err = yaml.Unmarshal([]byte(mdmd.yaml), &p)
if err != nil {
return nil, errors.Wrap(err, "unable to parse "+f.FullPath)
@@ -151,7 +161,10 @@ func ReadPolicies() ([]*Document, error) {
for _, f := range files {
p := &Document{}
mdmd := loadMDMD(f.FullPath)
mdmd, err := loadMDMD(f.FullPath)
if err != nil {
return nil, err
}
err = yaml.Unmarshal([]byte(mdmd.yaml), &p)
if err != nil {
return nil, errors.Wrap(err, "unable to parse "+f.FullPath)
@@ -171,21 +184,20 @@ type metadataMarkdown struct {
body string
}
func loadMDMD(path string) metadataMarkdown {
func loadMDMD(path string) (*metadataMarkdown, error) {
bytes, err := ioutil.ReadFile(path)
if err != nil {
panic(err)
}
content := string(bytes)
components := strings.Split(content, "---")
if components[0] == "" && (len(components) > 1) {
components = components[1:]
}
if len(components) == 1 {
panic(fmt.Sprintf("Malformed metadata markdown in %s, must be of the form: YAML\\n---\\nmarkdown content", path))
return nil, errors.New(fmt.Sprintf("Malformed metadata markdown in %s, must be of the form: YAML\\n---\\nmarkdown content", path))
}
yaml := components[0]
item := components[0]
body := strings.Join(components[1:], "---")
return metadataMarkdown{yaml, body}
return &metadataMarkdown{item, body}, nil
}

238
internal/model/fs_test.go Normal file
View File

@@ -0,0 +1,238 @@
package model
import (
"fmt"
"os"
"reflect"
"testing"
"github.com/strongdm/comply/internal/path"
"github.com/strongdm/comply/internal/util"
)
type ReadFiles struct{}
func beforeEach() {
util.MockConfig()
}
func TestReadFiles(t *testing.T) {
util.ExecuteTests(t, reflect.TypeOf(ReadFiles{}), beforeEach, nil)
}
// TestReadNarratives calls model.ReadNarratives checking for a valid return value.
func (tg ReadFiles) TestReadNarratives(t *testing.T) {
filePath := fmt.Sprintf("%s/narratives/control.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
path.Narratives = func() ([]path.File, error) {
return []path.File{
{FullPath: filePath, Info: fileInfo},
}, nil
}
documents, err := ReadNarratives()
if err != nil {
t.Fatalf(`ReadNarratives() returned an error %v`, err)
}
if len(documents) != 1 {
t.Fatal(`Invalid number of documents`)
}
if documents[0].FullPath != filePath {
t.Fatalf(`Invalid document path %s`, documents[0].FullPath)
}
}
// TestReadNarrativesWhenThereAreNoNarratives calls model.ReadNarratives checking for a valid return when
// there are no narratives to process
func (tg ReadFiles) TestReadNarrativesWhenThereAreNoNarratives(t *testing.T) {
path.Narratives = func() ([]path.File, error) {
return []path.File{}, nil
}
documents, err := ReadNarratives()
if err != nil {
t.Fatalf(`ReadNarratives() returned an error %v`, err)
}
if len(documents) != 0 {
t.Fatal(`Invalid number of documents`)
}
}
// TestReadNarrativesFailsWhenInvalidNarrative calls model.ReadNarratives checking for an error return when
// there is an invalid narrative
func (tg ReadFiles) TestReadNarrativesFailsWhenInvalidNarrative(t *testing.T) {
path.Narratives = func() ([]path.File, error) {
filePath := fmt.Sprintf("%s/../fixtures/narratives/invalid-control.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
return []path.File{
{FullPath: filePath, Info: fileInfo},
}, nil
}
_, err := ReadNarratives()
if err == nil {
t.Fatal(`ReadNarratives() was expected to fail`)
}
}
// TestReadProcedures calls model.ReadProcedures checking for a valid return value.
func (tg ReadFiles) TestReadProcedures(t *testing.T) {
filePath := fmt.Sprintf("%s/procedures/workstation.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
path.Procedures = func() ([]path.File, error) {
return []path.File{
{FullPath: filePath, Info: fileInfo},
}, nil
}
documents, err := ReadProcedures()
if err != nil {
t.Fatalf(`ReadProcedures() returned an error %v`, err)
}
if len(documents) != 1 {
t.Fatal(`Invalid number of documents`)
}
if documents[0].FullPath != filePath {
t.Fatalf(`Invalid document path %s`, documents[0].FullPath)
}
}
// TestReadProceduresWhenThereAreNoProcedures calls model.ReadProcedures checking for a valid return when
// there are no procedures to process
func (tg ReadFiles) TestReadProceduresWhenThereAreNoProcedures(t *testing.T) {
path.Procedures = func() ([]path.File, error) {
return []path.File{}, nil
}
documents, err := ReadProcedures()
if err != nil {
t.Fatalf(`ReadProcedures() returned an error %v`, err)
}
if len(documents) != 0 {
t.Fatal(`Invalid number of documents`)
}
}
// TestReadProceduresFailsWhenInvalidProcedure calls model.ReadProcedures checking for an error return when
// there is an invalid procedure
func (tg ReadFiles) TestReadProceduresFailsWhenInvalidProcedure(t *testing.T) {
path.Procedures = func() ([]path.File, error) {
filePath := fmt.Sprintf("%s/../fixtures/procedures/invalid-workstation.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
return []path.File{
{FullPath: filePath, Info: fileInfo},
}, nil
}
_, err := ReadProcedures()
if err == nil {
t.Fatal(`ReadProcedures() was expected to fail`, err)
}
}
// TestReadPolicies calls model.ReadPolicies checking for a valid return value.
func (tg ReadFiles) TestReadPolicies(t *testing.T) {
filePath := fmt.Sprintf("%s/policies/access.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
path.Policies = func() ([]path.File, error) {
return []path.File{
{FullPath: filePath, Info: fileInfo},
}, nil
}
documents, err := ReadPolicies()
if err != nil {
t.Fatalf(`ReadPolicies() returned an error %v`, err)
}
if len(documents) != 1 {
t.Fatal(`Invalid number of documents`)
}
if documents[0].FullPath != filePath {
t.Fatalf(`Invalid document path %s`, documents[0].FullPath)
}
}
// TestReadPoliciesWhenThereAreNoPolicies calls model.ReadPolicies checking for a valid return when
// there are no policies to process
func (tg ReadFiles) TestReadPoliciesWhenThereAreNoPolicies(t *testing.T) {
path.Policies = func() ([]path.File, error) {
return []path.File{}, nil
}
documents, err := ReadPolicies()
if err != nil {
t.Fatalf(`ReadPolicies() returned an error %v`, err)
}
if len(documents) != 0 {
t.Fatal(`Invalid number of documents`)
}
}
// TestReadPoliciesFailsWhenInvalidPolicy calls model.ReadPolicies checking for an error return when
// there is an invalid policy
func (tg ReadFiles) TestReadPoliciesFailsWhenInvalidPolicy(t *testing.T) {
path.Policies = func() ([]path.File, error) {
filePath := fmt.Sprintf("%s/../fixtures/policies/invalid-access.md", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
return []path.File{
{FullPath: filePath, Info: fileInfo},
}, nil
}
_, err := ReadPolicies()
if err == nil {
t.Fatal(`ReadPolicies() was expected to fail`, err)
}
}
// TestReadStandards calls model.ReadStandards checking for a valid return value.
func (tg ReadFiles) TestReadStandards(t *testing.T) {
filePath := fmt.Sprintf("%s/standards/TSC-2017.yml", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
path.Standards = func() ([]path.File, error) {
return []path.File{
{FullPath: filePath, Info: fileInfo},
}, nil
}
documents, err := ReadStandards()
if err != nil {
t.Fatalf(`ReadStandards() returned an error %v`, err)
}
if len(documents) != 1 {
t.Fatal(`Invalid number of documents`)
}
}
// TestReadStandardsWhenThereAreNoStandards calls model.ReadStandards checking for a valid return when
// there are no standards to process
func (tg ReadFiles) TestReadStandardsWhenThereAreNoStandards(t *testing.T) {
path.Standards = func() ([]path.File, error) {
return []path.File{}, nil
}
documents, err := ReadStandards()
if err != nil {
t.Fatalf(`ReadStandards() returned an error %v`, err)
}
if len(documents) != 0 {
t.Fatal(`Invalid number of documents`)
}
}
// TestReadStandardsFailsWhenInvalidStandard calls model.ReadStandards checking for an error return when
// there is an invalid standard
func (tg ReadFiles) TestReadStandardsFailsWhenInvalidStandard(t *testing.T) {
path.Standards = func() ([]path.File, error) {
filePath := fmt.Sprintf("%s/../fixtures/standards/invalid-standard.yml", util.GetRootPath())
fileInfo, _ := os.Lstat(filePath)
return []path.File{
{FullPath: filePath, Info: fileInfo},
}, nil
}
_, err := ReadStandards()
if err == nil {
t.Fatal(`ReadStandards() was expected to fail`, err)
}
}

View File

@@ -9,36 +9,32 @@ import (
func TestMarshal(t *testing.T) {
d := Data{
Tickets: []*Ticket{
&Ticket{
{
ID: "t1",
},
},
Audits: []*Audit{
&Audit{
{
ID: "a1",
},
},
Procedures: []*Procedure{
&Procedure{
Code: "pro1",
{
ID: "pro1",
},
},
Policies: []*Policy{
&Policy{
Policies: []*Document{
{
Name: "pol1",
},
},
}
m, _ := json.Marshal(d)
encoded := string(m)
if !strings.Contains(encoded, "t1") ||
!strings.Contains(encoded, "a1") ||
!strings.Contains(encoded, "pro1") ||
!strings.Contains(encoded, "pol1") {
t.Error("identifier not found in marshalled string")
}
}

View File

@@ -16,22 +16,22 @@ type File struct {
}
// Standards lists all standard files.
func Standards() ([]File, error) {
var Standards = func() ([]File, error) {
return filesFor("standards", "yml")
}
// Narratives lists all narrative files.
func Narratives() ([]File, error) {
var Narratives = func() ([]File, error) {
return filesFor("narratives", "md")
}
// Policies lists all policy files.
func Policies() ([]File, error) {
var Policies = func() ([]File, error) {
return filesFor("policies", "md")
}
// Procedures lists all procedure files.
func Procedures() ([]File, error) {
var Procedures = func() ([]File, error) {
return filesFor("procedures", "md")
}

View File

@@ -135,9 +135,12 @@ func preprocessDoc(data *renderData, pol *model.Document, fullPath string) error
%% %s
---
header-includes: yes
head-content: "%s"
foot-content: "%s confidential %d"
header-includes: |
\usepackage{fancyhdr}
\pagestyle{fancy}
\fancyhead{}
\fancyhead[RO,RE]{%s}
\fancyfoot[LO,LE]{%s confidential %d}
---
%s

View File

@@ -64,7 +64,7 @@ func dockerPandoc(outputFilename string, errOutputCh chan error) {
errOutputCh <- errors.Wrap(err, "unable to remove container")
return
}
errOutputCh <-nil
errOutputCh <- nil
}()
err = cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{})
@@ -78,7 +78,7 @@ func dockerPandoc(outputFilename string, errOutputCh chan error) {
if resultValue.StatusCode != 0 {
err = <-chanErr
errOutputCh <-errors.Wrap(err, "error awaiting Docker container")
errOutputCh <- errors.Wrap(err, "error awaiting Docker container")
return
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,52 @@
package util
import (
"fmt"
"io/ioutil"
"path/filepath"
"reflect"
"runtime"
"testing"
"github.com/strongdm/comply/internal/config"
"gopkg.in/yaml.v2"
)
type TestFixture func()
func ExecuteTests(t *testing.T, testGroupType reflect.Type, beforeEach TestFixture, afterEach TestFixture) {
testGroup := reflect.New(testGroupType).Elem().Interface()
for i := 0; i < testGroupType.NumMethod(); i++ {
m := testGroupType.Method(i)
t.Run(m.Name, func(t *testing.T) {
if beforeEach != nil {
beforeEach()
}
in := []reflect.Value{reflect.ValueOf(testGroup), reflect.ValueOf(t)}
m.Func.Call(in)
if afterEach != nil {
afterEach()
}
})
}
}
func MockConfig() {
config.Config = func() *config.Project {
p := config.Project{}
cfgBytes, _ := ioutil.ReadFile(filepath.Join(GetRootPath(), "comply.yml.example"))
err := yaml.Unmarshal(cfgBytes, &p)
if err != nil {
return nil
}
return &p
}
}
func GetRootPath() string {
_, fileName, _, _ := runtime.Caller(0)
fileDir := filepath.Dir(fileName)
return fmt.Sprintf("%s/../../example", fileDir)
}