diff --git a/cmd/constant.go b/cmd/constant.go index fb73fa9e..96d53466 100644 --- a/cmd/constant.go +++ b/cmd/constant.go @@ -16,4 +16,7 @@ var verbose = false var version = false var prettyPrint = false +// can be either "" (off), "extract" or "process" +var frontMatter = "" + var completedSuccessfully = false diff --git a/cmd/evalute_sequence_command.go b/cmd/evalute_sequence_command.go index 376cca6e..dcc82f10 100644 --- a/cmd/evalute_sequence_command.go +++ b/cmd/evalute_sequence_command.go @@ -103,6 +103,26 @@ func evaluateSequence(cmd *cobra.Command, args []string) error { return errors.New("Cannot pass files in when using null-input flag") } + if frontMatter != "" { + frontMatterHandler := yqlib.NewFrontMatterHandler(args[firstFileIndex]) + err = frontMatterHandler.Split() + if err != nil { + return err + } + args[firstFileIndex] = frontMatterHandler.GetYamlFrontMatterFilename() + + if frontMatter == "process" { + reader, err := os.Open(frontMatterHandler.GetContentFilename()) // #nosec + if err != nil { + return err + } + printer.SetAppendix(reader) + defer yqlib.SafelyCloseReader(reader) + } + defer frontMatterHandler.CleanUp() + + } + switch len(args) { case 0: if pipingStdIn { diff --git a/cmd/root.go b/cmd/root.go index 3132d0e0..b860547c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -54,6 +54,7 @@ See https://mikefarah.gitbook.io/yq/ for detailed documentation and examples.`, rootCmd.PersistentFlags().BoolVarP(&forceColor, "colors", "C", false, "force print with colors") rootCmd.PersistentFlags().BoolVarP(&forceNoColor, "no-colors", "M", false, "force print with no colors") + rootCmd.PersistentFlags().StringVarP(&frontMatter, "front-matter", "f", "", "(extract|process) first input as yaml front-matter. Extract will pull out the yaml content, process will run the expression against the yaml content, leaving the remaining data in-tact") rootCmd.AddCommand( createEvaluateSequenceCommand(), createEvaluateAllCommand(), diff --git a/examples/front-matter.yaml b/examples/front-matter.yaml new file mode 100644 index 00000000..7fb3e974 --- /dev/null +++ b/examples/front-matter.yaml @@ -0,0 +1,5 @@ +--- +a: fruit +b: banana +--- +content: cool diff --git a/pkg/yqlib/file_utils.go b/pkg/yqlib/file_utils.go index 00cbdbdc..7668a8e1 100644 --- a/pkg/yqlib/file_utils.go +++ b/pkg/yqlib/file_utils.go @@ -2,6 +2,7 @@ package yqlib import ( "io" + "io/ioutil" "os" ) @@ -23,6 +24,14 @@ func safelyRenameFile(from string, to string) { } } +func tryRemoveFile(filename string) { + log.Debug("Removing temp file: %v", filename) + removeErr := os.Remove(filename) + if removeErr != nil { + log.Errorf("Failed to remove temp file: %v", filename) + } +} + // thanks https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang func copyFileContents(src, dst string) (err error) { // ignore CWE-22 gosec issue - that's more targetted for http based apps that run in a public directory, @@ -44,6 +53,13 @@ func copyFileContents(src, dst string) (err error) { return out.Sync() } +func SafelyCloseReader(reader io.Reader) { + switch reader := reader.(type) { + case *os.File: + safelyCloseFile(reader) + } +} + func safelyCloseFile(file *os.File) { err := file.Close() if err != nil { @@ -51,3 +67,22 @@ func safelyCloseFile(file *os.File) { log.Error(err.Error()) } } + +func createTempFile() (*os.File, error) { + _, err := os.Stat(os.TempDir()) + if os.IsNotExist(err) { + err = os.Mkdir(os.TempDir(), 0700) + if err != nil { + return nil, err + } + } else if err != nil { + return nil, err + } + + file, err := ioutil.TempFile("", "temp") + if err != nil { + return nil, err + } + + return file, err +} diff --git a/pkg/yqlib/front_matter.go b/pkg/yqlib/front_matter.go new file mode 100644 index 00000000..16c39a5e --- /dev/null +++ b/pkg/yqlib/front_matter.go @@ -0,0 +1,96 @@ +package yqlib + +import ( + "bufio" + "io" + "os" +) + +type frontMatterHandler interface { + Split() error + GetYamlFrontMatterFilename() string + GetContentFilename() string + CleanUp() +} + +type frontMatterHandlerImpl struct { + originalFilename string + yamlFrontMatterFilename string + contentFilename string +} + +func NewFrontMatterHandler(originalFilename string) frontMatterHandler { + return &frontMatterHandlerImpl{originalFilename, "", ""} +} + +func (f *frontMatterHandlerImpl) GetYamlFrontMatterFilename() string { + return f.yamlFrontMatterFilename +} + +func (f *frontMatterHandlerImpl) GetContentFilename() string { + return f.contentFilename +} + +func (f *frontMatterHandlerImpl) CleanUp() { + tryRemoveFile(f.yamlFrontMatterFilename) + tryRemoveFile(f.contentFilename) +} + +// Splits the given file by yaml front matter +// yaml content will be saved to first temporary file +// remaining content will be saved to second temporary file +func (f *frontMatterHandlerImpl) Split() error { + var reader io.Reader + var err error + if f.originalFilename == "-" { + reader = bufio.NewReader(os.Stdin) + } else { + reader, err = os.Open(f.originalFilename) // #nosec + if err != nil { + return err + } + } + + yamlTempFile, err := createTempFile() + if err != nil { + return err + } + f.yamlFrontMatterFilename = yamlTempFile.Name() + log.Debug("yamlTempFile: %v", yamlTempFile.Name()) + + contentTempFile, err := createTempFile() + if err != nil { + return err + } + f.contentFilename = contentTempFile.Name() + log.Debug("contentTempFile: %v", contentTempFile.Name()) + + scanner := bufio.NewScanner(reader) + + lineCount := 0 + yamlContentBlock := true + + for scanner.Scan() { + line := scanner.Text() + + if lineCount > 0 && line == "---" { + //we've finished reading the yaml content + yamlContentBlock = false + } + if yamlContentBlock { + _, err = yamlTempFile.Write([]byte(line + "\n")) + } else { + _, err = contentTempFile.Write([]byte(line + "\n")) + } + if err != nil { + return err + } + lineCount = lineCount + 1 + } + + safelyCloseFile(yamlTempFile) + safelyCloseFile(contentTempFile) + + return scanner.Err() + +} diff --git a/pkg/yqlib/printer.go b/pkg/yqlib/printer.go index 9521da52..693b99af 100644 --- a/pkg/yqlib/printer.go +++ b/pkg/yqlib/printer.go @@ -13,8 +13,10 @@ type Printer interface { PrintedAnything() bool SetPrintLeadingSeperator(bool) - // preamble yaml content SetPreamble(reader io.Reader) + + //e.g. when given a front-matter doc, like jekyll + SetAppendix(reader io.Reader) } type resultsPrinter struct { @@ -30,6 +32,7 @@ type resultsPrinter struct { printedMatches bool treeNavigator DataTreeNavigator preambleReader io.Reader + appendixReader io.Reader } func NewPrinter(writer io.Writer, outputToJSON bool, unwrapScalar bool, colorsEnabled bool, indent int, printDocSeparators bool) Printer { @@ -56,6 +59,10 @@ func (p *resultsPrinter) SetPreamble(reader io.Reader) { p.preambleReader = reader } +func (p *resultsPrinter) SetAppendix(reader io.Reader) { + p.appendixReader = reader +} + func (p *resultsPrinter) PrintedAnything() bool { return p.printedMatches } @@ -139,5 +146,14 @@ func (p *resultsPrinter) PrintResults(matchingNodes *list.List) error { p.previousDocIndex = mappedDoc.Document } + if p.appendixReader != nil && !p.outputToJSON { + log.Debug("Piping appendix reader...") + betterReader := bufio.NewReader(p.appendixReader) + _, err := io.Copy(bufferedWriter, betterReader) + if err != nil { + return err + } + } + return nil } diff --git a/pkg/yqlib/write_in_place_handler.go b/pkg/yqlib/write_in_place_handler.go index 02a5d38d..533854cc 100644 --- a/pkg/yqlib/write_in_place_handler.go +++ b/pkg/yqlib/write_in_place_handler.go @@ -1,7 +1,6 @@ package yqlib import ( - "io/ioutil" "os" ) @@ -21,27 +20,21 @@ func NewWriteInPlaceHandler(inputFile string) writeInPlaceHandler { } func (w *writeInPlaceHandlerImpl) CreateTempFile() (*os.File, error) { + file, err := createTempFile() + + if err != nil { + return nil, err + } info, err := os.Stat(w.inputFilename) if err != nil { return nil, err } - _, err = os.Stat(os.TempDir()) - if os.IsNotExist(err) { - err = os.Mkdir(os.TempDir(), 0700) - if err != nil { - return nil, err - } - } else if err != nil { - return nil, err - } + err = os.Chmod(file.Name(), info.Mode()) - file, err := ioutil.TempFile("", "temp") if err != nil { return nil, err } - - err = os.Chmod(file.Name(), info.Mode()) - log.Debug("writing to tempfile: %v", file.Name()) + log.Debug("WriteInPlaceHandler: writing to tempfile: %v", file.Name()) w.tempFile = file return file, err } @@ -50,13 +43,9 @@ func (w *writeInPlaceHandlerImpl) FinishWriteInPlace(evaluatedSuccessfully bool) log.Debug("Going to write-inplace, evaluatedSuccessfully=%v, target=%v", evaluatedSuccessfully, w.inputFilename) safelyCloseFile(w.tempFile) if evaluatedSuccessfully { - log.Debug("moved temp file to target") + log.Debug("Moving temp file to target") safelyRenameFile(w.tempFile.Name(), w.inputFilename) } else { - log.Debug("removed temp file") - removeErr := os.Remove(w.tempFile.Name()) - if removeErr != nil { - log.Errorf("failed removing temp file: %s", w.tempFile.Name()) - } + tryRemoveFile(w.tempFile.Name()) } }