Initial commit

internal/render/controller.go

@@ -0,0 +1,142 @@
+package render
+
+import (
+	"fmt"
+	"sort"
+	"time"
+
+	"github.com/strongdm/comply/internal/config"
+	"github.com/strongdm/comply/internal/model"
+)
+
+type project struct {
+	OrganizationName string
+	Name             string
+}
+
+type stats struct {
+	ControlsTotal     int
+	ControlsSatisfied int
+
+	ProcessTotal      int
+	ProcessOpen       int
+	ProcessOldestDays int
+
+	AuditOpen   int
+	AuditClosed int
+	AuditTotal  int
+}
+
+type renderData struct {
+	// duplicates Project.OrganizationName
+	Name       string
+	Project    *project
+	Stats      *stats
+	Narratives []*model.Narrative
+	Policies   []*model.Policy
+	Procedures []*model.Procedure
+	Standards  []*model.Standard
+	Tickets    []*model.Ticket
+	Controls   []*control
+}
+
+type control struct {
+	Standard    string
+	ControlKey  string
+	Name        string
+	Description string
+	Satisfied   bool
+	SatisfiedBy []string
+}
+
+func load() (*model.Data, *renderData, error) {
+	modelData, err := model.ReadData()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	cfg := config.Config()
+	project := &project{
+		OrganizationName: cfg.Name,
+		Name:             fmt.Sprintf("%s Compliance Program", cfg.Name),
+	}
+
+	satisfied := model.ControlsSatisfied(modelData)
+	controls := make([]*control, 0)
+	for _, standard := range modelData.Standards {
+		for key, c := range standard.Controls {
+			satisfactions, ok := satisfied[key]
+			satisfied := ok && len(satisfactions) > 0
+			controls = append(controls, &control{
+				Standard:    standard.Name,
+				ControlKey:  key,
+				Name:        c.Name,
+				Description: c.Description,
+				Satisfied:   satisfied,
+				SatisfiedBy: satisfactions,
+			})
+		}
+	}
+	sort.Slice(controls, func(i, j int) bool {
+		return controls[i].ControlKey < controls[j].ControlKey
+	})
+
+	rd := &renderData{}
+	rd.Narratives = modelData.Narratives
+	rd.Policies = modelData.Policies
+	rd.Procedures = modelData.Procedures
+	rd.Standards = modelData.Standards
+	rd.Tickets = modelData.Tickets
+	rd.Project = project
+	rd.Name = project.OrganizationName
+	rd.Controls = controls
+	return modelData, rd, nil
+}
+
+func loadWithStats() (*model.Data, *renderData, error) {
+	modelData, renderData, err := load()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	addStats(modelData, renderData)
+	return modelData, renderData, nil
+}
+
+func addStats(modelData *model.Data, renderData *renderData) {
+	stats := &stats{}
+
+	satisfied := model.ControlsSatisfied(modelData)
+
+	for _, std := range renderData.Standards {
+		stats.ControlsTotal += len(std.Controls)
+		for controlKey := range std.Controls {
+			if _, ok := satisfied[controlKey]; ok {
+				stats.ControlsSatisfied++
+			}
+		}
+	}
+
+	for _, t := range renderData.Tickets {
+		if t.Bool("audit") {
+			stats.AuditTotal++
+		}
+
+		if t.State == model.Open {
+			if t.Bool("process") {
+				stats.ProcessOpen++
+				if t.CreatedAt != nil {
+					age := int(time.Since(*t.CreatedAt).Hours() / float64(24))
+					if stats.ProcessOldestDays < age {
+						stats.ProcessOldestDays = age
+					}
+				}
+			}
+			if t.Bool("audit") {
+				stats.AuditOpen++
+			}
+		}
+	}
+
+	renderData.Stats = stats
+}

internal/render/doc.go

@@ -0,0 +1,4 @@
+/*
+Package render defines markdown preprocessors, HTML and PDF generation.
+*/
+package render

internal/render/html.go

@@ -0,0 +1,80 @@
+package render
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/yosssi/ace"
+)
+
+const websocketReloader = `<script>
+(function(){
+	var ws = new WebSocket("ws://localhost:5122/ws")
+	var connected = false
+	ws.onopen = function(e) {
+		connected = true
+	}
+	ws.onclose = function(e) {
+		// reload!
+		if (connected) {
+			window.location=window.location
+		}
+	}
+})()
+</script>`
+
+func html(output string, live bool, errCh chan error, wg *sync.WaitGroup) {
+	for {
+		files, err := ioutil.ReadDir(filepath.Join(".", "templates"))
+		if err != nil {
+			errCh <- errors.Wrap(err, "unable to open template directory")
+			return
+		}
+
+		_, data, err := loadWithStats()
+		if err != nil {
+			errCh <- errors.Wrap(err, "unable to load data")
+			return
+		}
+
+		for _, fileInfo := range files {
+			if !strings.HasSuffix(fileInfo.Name(), ".ace") {
+				continue
+			}
+
+			basename := strings.Replace(fileInfo.Name(), ".ace", "", -1)
+			w, err := os.Create(filepath.Join(output, fmt.Sprintf("%s.html", basename)))
+			if err != nil {
+				errCh <- errors.Wrap(err, "unable to create HTML file")
+				return
+			}
+
+			tpl, err := ace.Load("", filepath.Join("templates", basename), aceOpts)
+			if err != nil {
+				w.Write([]byte("<htmL><body>template error</body></html>"))
+				fmt.Println(err)
+			}
+
+			err = tpl.Execute(w, data)
+			if err != nil {
+				w.Write([]byte("<htmL><body>template error</body></html>"))
+				fmt.Println(err)
+			}
+
+			if live {
+				w.Write([]byte(websocketReloader))
+			}
+			w.Close()
+		}
+		if !live {
+			wg.Done()
+			return
+		}
+		<-subscribe()
+	}
+}

internal/render/narrative.go

@@ -0,0 +1,185 @@
+package render
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"text/template"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"github.com/strongdm/comply/internal/config"
+	"github.com/strongdm/comply/internal/model"
+)
+
+// TODO: refactor and eliminate duplication among narrative, policy renderers
+func renderNarrativeToDisk(wg *sync.WaitGroup, errOutputCh chan error, data *renderData, narrative *model.Narrative, live bool) {
+	// only files that have been touched
+	if !isNewer(narrative.FullPath, narrative.ModifiedAt) {
+		return
+	}
+	recordModified(narrative.FullPath, narrative.ModifiedAt)
+
+	ctx := context.Background()
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		errOutputCh <- errors.Wrap(err, "unable to read Docker environment")
+		return
+	}
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		errOutputCh <- errors.Wrap(err, "unable to get workding directory")
+		return
+	}
+
+	hc := &container.HostConfig{
+		Binds: []string{pwd + ":/source"},
+	}
+
+	wg.Add(1)
+	go func(p *model.Narrative) {
+		defer wg.Done()
+
+		if live {
+			rel, err := filepath.Rel(config.ProjectRoot(), p.FullPath)
+			if err != nil {
+				rel = p.FullPath
+			}
+			fmt.Printf("%s -> %s\n", rel, filepath.Join("output", p.OutputFilename))
+		}
+
+		outputFilename := p.OutputFilename
+		// save preprocessed markdown
+		err = preprocessNarrative(data, p, filepath.Join(".", "output", outputFilename+".md"))
+		if err != nil {
+			errOutputCh <- errors.Wrap(err, "unable to preprocess")
+			return
+		}
+
+		cmd := []string{"--smart", "--toc", "-N", "--template=/source/templates/default.latex", "-o",
+			fmt.Sprintf("/source/output/%s", outputFilename),
+			fmt.Sprintf("/source/output/%s.md", outputFilename)}
+
+		resp, err := cli.ContainerCreate(ctx, &container.Config{
+			Image: "strongdm/pandoc",
+			Cmd:   cmd},
+			hc, nil, "")
+
+		if err != nil {
+			errOutputCh <- errors.Wrap(err, "unable to create Docker container")
+			return
+		}
+
+		defer func() {
+			timeout := 2 * time.Second
+			cli.ContainerStop(ctx, resp.ID, &timeout)
+			err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})
+			if err != nil {
+				errOutputCh <- errors.Wrap(err, "unable to remove container")
+				return
+			}
+		}()
+
+		if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
+			errOutputCh <- errors.Wrap(err, "unable to start Docker container")
+			return
+		}
+
+		_, err = cli.ContainerWait(ctx, resp.ID)
+		if err != nil {
+			errOutputCh <- errors.Wrap(err, "error awaiting Docker container")
+			return
+		}
+
+		_, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true})
+		if err != nil {
+			errOutputCh <- errors.Wrap(err, "error reading Docker container logs")
+			return
+		}
+
+		// remove preprocessed markdown
+		err = os.Remove(filepath.Join(".", "output", outputFilename+".md"))
+		if err != nil {
+			errOutputCh <- err
+			return
+		}
+	}(narrative)
+}
+
+func preprocessNarrative(data *renderData, pol *model.Narrative, fullPath string) error {
+	cfg := config.Config()
+
+	var w bytes.Buffer
+	bodyTemplate, err := template.New("body").Parse(pol.Body)
+	if err != nil {
+		w.WriteString(fmt.Sprintf("# Error processing template:\n\n%s\n", err.Error()))
+	} else {
+		bodyTemplate.Execute(&w, data)
+	}
+	body := w.String()
+
+	revisionTable := ""
+	satisfiesTable := ""
+
+	// ||Date|Comment|
+	// |---+------|
+	// | 4 Jan 2018 | Initial Version |
+	// Table: Document history
+
+	if len(pol.Satisfies) > 0 {
+		rows := ""
+		for standard, keys := range pol.Satisfies {
+			rows += fmt.Sprintf("| %s | %s |\n", standard, strings.Join(keys, ", "))
+		}
+		satisfiesTable = fmt.Sprintf("|Standard|Controls Satisfied|\n|-------+--------------------------------------------|\n%s\nTable: Control satisfaction\n", rows)
+	}
+
+	if len(pol.Revisions) > 0 {
+		rows := ""
+		for _, rev := range pol.Revisions {
+			rows += fmt.Sprintf("| %s | %s |\n", rev.Date, rev.Comment)
+		}
+		revisionTable = fmt.Sprintf("|Date|Comment|\n|---+--------------------------------------------|\n%s\nTable: Document history\n", rows)
+	}
+
+	doc := fmt.Sprintf(`%% %s
+%% %s
+%% %s
+
+---
+header-includes: yes
+head-content: "%s"
+foot-content: "%s confidential %d"
+---
+
+%s
+
+%s
+
+\newpage
+%s`,
+		pol.Name,
+		cfg.Name,
+		fmt.Sprintf("%s %d", pol.ModifiedAt.Month().String(), pol.ModifiedAt.Year()),
+		pol.Name,
+		cfg.Name,
+		time.Now().Year(),
+		satisfiesTable,
+		revisionTable,
+		body,
+	)
+	err = ioutil.WriteFile(fullPath, []byte(doc), os.FileMode(0644))
+	if err != nil {
+		return errors.Wrap(err, "unable to write preprocessed narrative to disk")
+	}
+	return nil
+}

internal/render/pdf.go

@@ -0,0 +1,49 @@
+package render
+
+import (
+	"sync"
+
+	"github.com/pkg/errors"
+	"github.com/strongdm/comply/internal/model"
+)
+
+func pdf(output string, live bool, errCh chan error, wg *sync.WaitGroup) {
+	var pdfWG sync.WaitGroup
+
+	errOutputCh := make(chan error)
+
+	for {
+		_, data, err := loadWithStats()
+		if err != nil {
+			errCh <- errors.Wrap(err, "unable to load data")
+			return
+		}
+
+		policies, err := model.ReadPolicies()
+		if err != nil {
+			errCh <- errors.Wrap(err, "unable to read policies")
+			return
+		}
+		for _, policy := range policies {
+			renderPolicyToDisk(&pdfWG, errOutputCh, data, policy, live)
+		}
+
+		narratives, err := model.ReadNarratives()
+		if err != nil {
+			errCh <- errors.Wrap(err, "unable to read narratives")
+			return
+		}
+
+		for _, narrative := range narratives {
+			renderNarrativeToDisk(&pdfWG, errOutputCh, data, narrative, live)
+		}
+
+		pdfWG.Wait()
+
+		if !live {
+			wg.Done()
+			return
+		}
+		<-subscribe()
+	}
+}

internal/render/policy.go

@@ -0,0 +1,183 @@
+package render
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"text/template"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/client"
+	"github.com/pkg/errors"
+	"github.com/strongdm/comply/internal/config"
+	"github.com/strongdm/comply/internal/model"
+)
+
+// TODO: refactor and eliminate duplication among narrative, policy renderers
+func renderPolicyToDisk(wg *sync.WaitGroup, errOutputCh chan error, data *renderData, policy *model.Policy, live bool) {
+	// only files that have been touched
+	if !isNewer(policy.FullPath, policy.ModifiedAt) {
+		return
+	}
+	recordModified(policy.FullPath, policy.ModifiedAt)
+
+	ctx := context.Background()
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		errOutputCh <- errors.Wrap(err, "unable to read Docker environment")
+		return
+	}
+
+	pwd, err := os.Getwd()
+	if err != nil {
+		errOutputCh <- errors.Wrap(err, "unable to get workding directory")
+		return
+	}
+
+	hc := &container.HostConfig{
+		Binds: []string{pwd + ":/source"},
+	}
+
+	wg.Add(1)
+	go func(p *model.Policy) {
+		defer wg.Done()
+
+		if live {
+			rel, err := filepath.Rel(config.ProjectRoot(), p.FullPath)
+			if err != nil {
+				rel = p.FullPath
+			}
+			fmt.Printf("%s -> %s\n", rel, filepath.Join("output", p.OutputFilename))
+		}
+
+		outputFilename := p.OutputFilename
+		// save preprocessed markdown
+		err = preprocessPolicy(data, p, filepath.Join(".", "output", outputFilename+".md"))
+		if err != nil {
+			errOutputCh <- errors.Wrap(err, "unable to preprocess")
+			return
+		}
+
+		resp, err := cli.ContainerCreate(ctx, &container.Config{
+			Image: "strongdm/pandoc",
+			Cmd: []string{"--smart", "--toc", "-N", "--template=/source/templates/default.latex", "-o",
+				fmt.Sprintf("/source/output/%s", outputFilename),
+				fmt.Sprintf("/source/output/%s.md", outputFilename),
+			},
+		}, hc, nil, "")
+		if err != nil {
+			errOutputCh <- errors.Wrap(err, "unable to create Docker container")
+			return
+		}
+
+		defer func() {
+			timeout := 2 * time.Second
+			cli.ContainerStop(ctx, resp.ID, &timeout)
+			err := cli.ContainerRemove(ctx, resp.ID, types.ContainerRemoveOptions{Force: true})
+			if err != nil {
+				errOutputCh <- errors.Wrap(err, "unable to remove container")
+				return
+			}
+		}()
+
+		if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
+			errOutputCh <- errors.Wrap(err, "unable to start Docker container")
+			return
+		}
+
+		_, err = cli.ContainerWait(ctx, resp.ID)
+		if err != nil {
+			errOutputCh <- errors.Wrap(err, "error awaiting Docker container")
+			return
+		}
+
+		_, err = cli.ContainerLogs(ctx, resp.ID, types.ContainerLogsOptions{ShowStdout: true})
+		if err != nil {
+			errOutputCh <- errors.Wrap(err, "error reading Docker container logs")
+			return
+		}
+
+		// remove preprocessed markdown
+		err = os.Remove(filepath.Join(".", "output", outputFilename+".md"))
+		if err != nil {
+			errOutputCh <- err
+			return
+		}
+	}(policy)
+}
+
+func preprocessPolicy(data *renderData, pol *model.Policy, fullPath string) error {
+	cfg := config.Config()
+
+	var w bytes.Buffer
+	bodyTemplate, err := template.New("body").Parse(pol.Body)
+	if err != nil {
+		w.WriteString(fmt.Sprintf("# Error processing template:\n\n%s\n", err.Error()))
+	} else {
+		bodyTemplate.Execute(&w, data)
+	}
+	body := w.String()
+
+	revisionTable := ""
+	satisfiesTable := ""
+
+	// ||Date|Comment|
+	// |---+------|
+	// | 4 Jan 2018 | Initial Version |
+	// Table: Document history
+
+	if len(pol.Satisfies) > 0 {
+		rows := ""
+		for standard, keys := range pol.Satisfies {
+			rows += fmt.Sprintf("| %s | %s |\n", standard, strings.Join(keys, ", "))
+		}
+		satisfiesTable = fmt.Sprintf("|Standard|Controls Satisfied|\n|-------+--------------------------------------------|\n%s\nTable: Control satisfaction\n", rows)
+	}
+
+	if len(pol.Revisions) > 0 {
+		rows := ""
+		for _, rev := range pol.Revisions {
+			rows += fmt.Sprintf("| %s | %s |\n", rev.Date, rev.Comment)
+		}
+		revisionTable = fmt.Sprintf("|Date|Comment|\n|---+--------------------------------------------|\n%s\nTable: Document history\n", rows)
+	}
+
+	doc := fmt.Sprintf(`%% %s
+%% %s
+%% %s
+
+---
+header-includes: yes
+head-content: "%s"
+foot-content: "%s confidential %d"
+---
+
+%s
+
+%s
+
+\newpage
+%s`,
+		pol.Name,
+		cfg.Name,
+		fmt.Sprintf("%s %d", pol.ModifiedAt.Month().String(), pol.ModifiedAt.Year()),
+		pol.Name,
+		cfg.Name,
+		time.Now().Year(),
+		satisfiesTable,
+		revisionTable,
+		body,
+	)
+	err = ioutil.WriteFile(fullPath, []byte(doc), os.FileMode(0644))
+	if err != nil {
+		return errors.Wrap(err, "unable to write preprocessed policy to disk")
+	}
+	return nil
+}

internal/render/site.go

@@ -0,0 +1,118 @@
+package render
+
+import (
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"github.com/gorilla/websocket"
+	"github.com/pkg/errors"
+	"github.com/skratchdot/open-golang/open"
+	"github.com/yosssi/ace"
+)
+
+var upgrader = websocket.Upgrader{
+	ReadBufferSize:  1024,
+	WriteBufferSize: 1024,
+	CheckOrigin: func(r *http.Request) bool {
+		return true
+	},
+}
+
+var aceOpts = &ace.Options{
+	DynamicReload: true,
+	Indent:        "  ",
+}
+
+var watchChMu sync.Mutex
+var watchCh chan struct{}
+
+func subscribe() chan struct{} {
+	watchChMu.Lock()
+	defer watchChMu.Unlock()
+	if watchCh == nil {
+		watchCh = make(chan struct{})
+	}
+	return watchCh
+}
+
+func broadcast() {
+	watchChMu.Lock()
+	defer watchChMu.Unlock()
+	close(watchCh)
+	watchCh = nil
+}
+
+var lastModifiedMu sync.Mutex
+var lastModified = make(map[string]time.Time)
+
+func recordModified(path string, t time.Time) {
+	lastModifiedMu.Lock()
+	defer lastModifiedMu.Unlock()
+
+	previous, ok := lastModified[path]
+	if !ok || t.After(previous) {
+		lastModified[path] = t
+	}
+}
+
+func isNewer(path string, t time.Time) bool {
+	lastModifiedMu.Lock()
+	defer lastModifiedMu.Unlock()
+
+	previous, ok := lastModified[path]
+	if !ok {
+		return true
+	}
+
+	// is tested after previous? Then isNewer is true.
+	return t.After(previous)
+}
+
+// Build generates all PDF and HTML output to the target directory with optional live reload.
+func Build(output string, live bool) error {
+	err := os.RemoveAll(output)
+	if err != nil {
+		errors.Wrap(err, "unable to remove files from output directory")
+	}
+
+	err = os.MkdirAll(output, os.FileMode(0755))
+	if err != nil {
+		errors.Wrap(err, "unable to create output directory")
+	}
+
+	var wg sync.WaitGroup
+	errCh := make(chan error, 0)
+	wgCh := make(chan struct{})
+
+	if live {
+		watch(errCh)
+	}
+	// PDF
+	wg.Add(1)
+	go pdf(output, live, errCh, &wg)
+
+	// HTML
+	wg.Add(1)
+	go html(output, live, errCh, &wg)
+
+	// WG monitor
+	go func() {
+		wg.Wait()
+		close(wgCh)
+	}()
+
+	if live {
+		open.Run("output/index.html")
+	}
+
+	select {
+	case <-wgCh:
+		// success
+	case err := <-errCh:
+		return errors.Wrap(err, "error during build")
+	}
+
+	return nil
+}

internal/render/watch.go

@@ -0,0 +1,50 @@
+package render
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/gohugoio/hugo/watcher"
+)
+
+func watch(errCh chan error) {
+	b, err := watcher.New(300 * time.Millisecond)
+	if err != nil {
+		errCh <- err
+		return
+	}
+	b.Add("./templates/")
+	b.Add("./policies/")
+	b.Add("./procedures/")
+
+	b.Add("./.comply/")
+	b.Add("./.comply/cache")
+	b.Add("./.comply/cache/tickets")
+
+	go func() {
+		for {
+			select {
+			case e := <-b.Errors:
+				errCh <- e
+			case <-b.Events:
+				broadcast()
+			}
+		}
+	}()
+
+	serveWs := func(w http.ResponseWriter, r *http.Request) {
+		ws, err := upgrader.Upgrade(w, r, nil)
+		if err != nil {
+			errCh <- err
+			return
+		}
+		<-subscribe()
+		time.Sleep(500 * time.Millisecond)
+		ws.Close()
+	}
+
+	http.HandleFunc("/ws", serveWs)
+	go http.ListenAndServe("127.0.0.1:5122", nil)
+
+	return
+}