Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize output and flags #193

Merged
merged 44 commits into from
Oct 2, 2024
Merged
Show file tree
Hide file tree
Changes from 42 commits
Commits
Show all changes
44 commits
Select commit Hold shift + click to select a range
7e70c95
fixed
YairSlobodin1 Aug 26, 2024
19123f9
minor change
YairSlobodin1 Aug 26, 2024
20ab99f
Merge branch 'optimize' into read_sgs
YairSlobodin1 Aug 26, 2024
2b7f244
added synth prefix
YairSlobodin1 Aug 26, 2024
8203f78
unexported two functions
YairSlobodin1 Aug 26, 2024
5fbeefc
wip
YairSlobodin1 Aug 26, 2024
b228063
Merge branch 'optimize' into read_sgs
YairSlobodin1 Aug 26, 2024
ebe6148
Merge branch 'read_sgs' into output
YairSlobodin1 Aug 26, 2024
d17ce6d
Merge branch 'optimize' into read_sgs
YairSlobodin1 Aug 26, 2024
32f3104
Merge branch 'read_sgs' into output
YairSlobodin1 Aug 26, 2024
02030ac
updated
YairSlobodin1 Aug 26, 2024
47fe8bb
merge
YairSlobodin1 Aug 26, 2024
1371e59
Merge branch 'read_sgs' into output
YairSlobodin1 Aug 26, 2024
0c9cfe2
wip
YairSlobodin1 Aug 26, 2024
eebd628
template
YairSlobodin1 Aug 27, 2024
26473b3
done
YairSlobodin1 Aug 28, 2024
a005a82
Merge branch 'optimize' into read_sgs
YairSlobodin1 Sep 2, 2024
9692f06
Merge branch 'read_sgs' into output
YairSlobodin1 Sep 2, 2024
a683181
Merge branch 'optimize' into read_sgs
YairSlobodin1 Sep 8, 2024
77a3fe0
Merge branch 'read_sgs' into output
YairSlobodin1 Sep 8, 2024
a7f68f5
lint
YairSlobodin1 Sep 11, 2024
380c62e
fixed
YairSlobodin1 Sep 12, 2024
c22aad0
fixed
YairSlobodin1 Sep 12, 2024
81f6194
fixed
YairSlobodin1 Sep 12, 2024
197ed31
fixed
YairSlobodin1 Sep 12, 2024
f93c27c
Merge branch 'optimize' into read_sgs
YairSlobodin1 Sep 25, 2024
6be5ee0
fixed
YairSlobodin1 Sep 25, 2024
2f9991a
wip
YairSlobodin1 Sep 25, 2024
bf9438f
wip
YairSlobodin1 Sep 25, 2024
e3d4668
wip
YairSlobodin1 Sep 25, 2024
ebb4b3b
wip
YairSlobodin1 Sep 25, 2024
f34b9b7
fixed
YairSlobodin1 Sep 25, 2024
f228c0b
Merge branch 'optimize' into read_sgs
YairSlobodin1 Sep 25, 2024
d3c7703
Merge branch 'read_sgs' into output
YairSlobodin1 Sep 25, 2024
5001289
Merge branch 'optimize' into read_sgs
YairSlobodin1 Sep 29, 2024
a89e1fe
merge
YairSlobodin1 Sep 29, 2024
ca64cee
avoid code dup
YairSlobodin1 Sep 30, 2024
49f4a47
fixed
YairSlobodin1 Sep 30, 2024
766f737
fix output fmts
YairSlobodin1 Sep 30, 2024
3145c59
merge
YairSlobodin1 Oct 1, 2024
cc37469
comments wip
YairSlobodin1 Oct 1, 2024
7226233
fix review comments
YairSlobodin1 Oct 1, 2024
7cb3792
fix review comments
YairSlobodin1 Oct 2, 2024
be1ee94
revie comments
YairSlobodin1 Oct 2, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 23 additions & 1 deletion cmd/subcmds/optimize.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,14 @@ SPDX-License-Identifier: Apache-2.0

package subcmds

import "github.com/spf13/cobra"
import (
"fmt"

"github.com/spf13/cobra"

"github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
"github.com/np-guard/vpc-network-config-synthesis/pkg/optimize"
)

func NewOptimizeCommand(args *inArgs) *cobra.Command {
cmd := &cobra.Command{
Expand All @@ -14,7 +21,22 @@ func NewOptimizeCommand(args *inArgs) *cobra.Command {
Long: `optimization of existing SG (nACLS are not supported yet)`,
}

// sub cmds
cmd.AddCommand(NewOptimizeSGCommand(args))

return cmd
}

func optimization(cmd *cobra.Command, args *inArgs, newOptimizer func(ir.Collection, string) optimize.Optimizer, isSG bool) error {
cmd.SilenceUsage = true // if we got this far, flags are syntactically correct, so no need to print usage
collection, err := parseCollection(args, isSG)
if err != nil {
return fmt.Errorf("could not parse config file %v: %w", args.configFile, err)
}
optimizer := newOptimizer(collection, args.firewallName)
optimizedCollection, err := optimizer.Optimize()
if err != nil {
return err
}
return writeOutput(args, optimizedCollection, collection.VpcNames(), false)
}
14 changes: 0 additions & 14 deletions cmd/subcmds/optimizeOutput.go

This file was deleted.

23 changes: 5 additions & 18 deletions cmd/subcmds/optimizeSG.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,39 +6,26 @@ SPDX-License-Identifier: Apache-2.0
package subcmds

import (
"fmt"

"github.com/spf13/cobra"

"github.com/np-guard/vpc-network-config-synthesis/pkg/io/confio"
"github.com/np-guard/vpc-network-config-synthesis/pkg/optimize"
)

const sgNameFlag = "sg-name"

func NewOptimizeSGCommand(args *inArgs) *cobra.Command {
cmd := &cobra.Command{
Use: "sg",
Short: "OptimizeSG attempts to reduce the number of security group rules in a SG without changing the semantic.",
Long: `OptimizeSG attempts to reduce the number of security group rules in a SG without changing the semantic.`,
Args: cobra.NoArgs,
RunE: func(cmd *cobra.Command, _ []string) error {
return optimization(cmd, args)
return optimization(cmd, args, optimize.NewSGOptimizer, true)
},
}

cmd.Flags().StringVarP(&args.sgName, sgNameFlag, "s", "", "which SG to optimize")
_ = cmd.MarkFlagRequired(sgNameFlag) // Todo: delete this line. if sgName flag is not supplied - optimize all SGs
// flags
cmd.PersistentFlags().StringVarP(&args.firewallName, sgNameFlag, "n", "", "which security group to optimize")

return cmd
}

func optimization(cmd *cobra.Command, args *inArgs) error {
cmd.SilenceUsage = true // if we got this far, flags are syntactically correct, so no need to print usage
sgs, err := confio.ReadSGs(args.configFile)
if err != nil {
return fmt.Errorf("could not parse config file %v: %w", args.configFile, err)
}
if optimize.ReduceSGRules(sgs, args.sgName) != nil {
return err
}
return writeOptimizeOutput(args, sgs)
}
55 changes: 23 additions & 32 deletions cmd/subcmds/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,33 +16,27 @@ import (
"github.com/np-guard/vpc-network-config-synthesis/pkg/io/csvio"
"github.com/np-guard/vpc-network-config-synthesis/pkg/io/mdio"
"github.com/np-guard/vpc-network-config-synthesis/pkg/io/tfio"

"github.com/np-guard/vpc-network-config-synthesis/pkg/ir"
)

const defaultFilePermission = 0o644
const defaultDirectoryPermission = 0o755

func writeOutput(args *inArgs, collection ir.Collection, vpcNames []ir.ID) error {
if err := updateOutputFormat(args); err != nil {
return err
}
if args.outputDir != "" && args.outputFmt == apiOutputFormat {
return fmt.Errorf("-d cannot be used with format json")
}
func writeOutput(args *inArgs, collection ir.Collection, vpcNames []string, isSynth bool) error {
if args.outputDir != "" { // create the directory if needed
if err := os.MkdirAll(args.outputDir, defaultDirectoryPermission); err != nil {
return err
}
}
if err := writeLocals(args, collection, vpcNames); err != nil {
_, isACLCollection := collection.(*ir.ACLCollection)
zivnevo marked this conversation as resolved.
Show resolved Hide resolved
if err := writeLocals(args, vpcNames, isACLCollection); err != nil {
return err
}

var data *bytes.Buffer
var err error
if args.outputDir == "" {
if data, err = writeCollection(args, collection, ""); err != nil {
if data, err = writeCollection(args, collection, "", isSynth); err != nil {
return err
}
return writeToFile(args.outputFile, data)
Expand All @@ -55,7 +49,7 @@ func writeOutput(args *inArgs, collection ir.Collection, vpcNames []ir.ID) error
if args.prefix != "" {
args.outputFile = args.outputDir + "/" + args.prefix + "_" + suffix
}
if data, err = writeCollection(args, collection, vpc); err != nil {
if data, err = writeCollection(args, collection, vpc, isSynth); err != nil {
return err
}
if err := writeToFile(args.outputFile, data); err != nil {
Expand All @@ -65,9 +59,9 @@ func writeOutput(args *inArgs, collection ir.Collection, vpcNames []ir.ID) error
return nil
}

func writeCollection(args *inArgs, collection ir.Collection, vpc string) (*bytes.Buffer, error) {
func writeCollection(args *inArgs, collection ir.Collection, vpc string, isSynth bool) (*bytes.Buffer, error) {
var data bytes.Buffer
writer, err := pickWriter(args, &data)
writer, err := pickWriter(args, &data, isSynth)
if err != nil {
return nil, err
}
Expand All @@ -77,15 +71,7 @@ func writeCollection(args *inArgs, collection ir.Collection, vpc string) (*bytes
return &data, nil
}

func writeToFile(outputFile string, data *bytes.Buffer) error {
if outputFile == "" {
fmt.Println(data.String())
return nil
}
return os.WriteFile(outputFile, data.Bytes(), defaultFilePermission)
}

func pickWriter(args *inArgs, data *bytes.Buffer) (ir.Writer, error) {
func pickWriter(args *inArgs, data *bytes.Buffer, isSynth bool) (ir.Writer, error) {
w := bufio.NewWriter(data)
switch args.outputFmt {
case tfOutputFormat:
Expand All @@ -94,25 +80,30 @@ func pickWriter(args *inArgs, data *bytes.Buffer) (ir.Writer, error) {
return csvio.NewWriter(w), nil
case mdOutputFormat:
return mdio.NewWriter(w), nil
case apiOutputFormat:
return confio.NewWriter(w, args.configFile)
default:
return nil, fmt.Errorf("bad output format: %q", args.outputFmt)
case jsonOutputFormat:
if isSynth {
zivnevo marked this conversation as resolved.
Show resolved Hide resolved
return confio.NewWriter(w, args.configFile)
}
}
return nil, fmt.Errorf("bad output format: %q", args.outputFmt)
}

func writeLocals(args *inArgs, collection ir.Collection, vpcNames []ir.ID) error {
if !args.locals {
func writeToFile(outputFile string, data *bytes.Buffer) error {
if outputFile == "" {
fmt.Println(data.String())
return nil
}
if args.outputFmt != tfOutputFormat {
return fmt.Errorf("--locals flag requires setting the output format to tf")
return os.WriteFile(outputFile, data.Bytes(), defaultFilePermission)
}

func writeLocals(args *inArgs, vpcNames []ir.ID, isACL bool) error {
if !args.locals {
return nil
}

_, isACLCollection := collection.(*ir.ACLCollection)
var data *bytes.Buffer
var err error
if data, err = tfio.WriteLocals(vpcNames, isACLCollection); err != nil {
if data, err = tfio.WriteLocals(vpcNames, isACL); err != nil {
return err
}

Expand Down
6 changes: 3 additions & 3 deletions cmd/subcmds/outputFormat.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,11 @@ const (
tfOutputFormat = "tf"
csvOutputFormat = "csv"
mdOutputFormat = "md"
apiOutputFormat = "json"
jsonOutputFormat = "json"
defaultOutputFormat = csvOutputFormat
)

var outputFormats = []string{tfOutputFormat, csvOutputFormat, mdOutputFormat, apiOutputFormat}
var outputFormats = []string{tfOutputFormat, csvOutputFormat, mdOutputFormat, jsonOutputFormat}

func updateOutputFormat(args *inArgs) error {
var err error
Expand All @@ -40,7 +40,7 @@ func inferFormatUsingFilename(filename string) (string, error) {
case strings.HasSuffix(filename, ".md"):
return mdOutputFormat, nil
case strings.HasSuffix(filename, ".json"):
return apiOutputFormat, nil
return jsonOutputFormat, nil
default:
return "", fmt.Errorf("bad output format")
}
Expand Down
38 changes: 21 additions & 17 deletions cmd/subcmds/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,56 +14,60 @@ import (

const (
configFlag = "config"
specFlag = "spec"
outputFmtFlag = "format"
outputFileFlag = "output-file"
outputDirFlag = "output-dir"
prefixFlag = "prefix"
sgNameFlag = "sg-name"
singleACLFlag = "single"
localsFlag = "locals"
)

type inArgs struct {
configFile string
specFile string
outputFmt string
outputFile string
outputDir string
prefix string
sgName string
singleacl bool
locals bool
configFile string
specFile string
outputFmt string
outputFile string
outputDir string
prefix string
firewallName string
singleacl bool
locals bool
}

func NewRootCommand() *cobra.Command {
args := &inArgs{}

// allow PersistentPreRunE
cobra.EnableTraverseRunHooks = true

rootCmd := &cobra.Command{
Use: "vpcgen",
Short: "Tool for automatic synthesis of VPC network configurations",
Long: `Tool for automatic synthesis of VPC network configurations, namely Network ACLs and Security Groups.`,
PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
return validateFlags(args)
},
}

// flags
rootCmd.PersistentFlags().StringVarP(&args.configFile, configFlag, "c", "",
"JSON file containing a configuration object of existing resources")
rootCmd.PersistentFlags().StringVarP(&args.outputFmt, outputFmtFlag, "f", "", "Output format; "+mustBeOneOf(outputFormats))
rootCmd.PersistentFlags().StringVarP(&args.outputFile, outputFileFlag, "o", "", "Write all generated resources to the specified file.")
rootCmd.PersistentFlags().StringVarP(&args.outputDir, outputDirFlag, "d", "",
"Write generated resources to files in the specified directory, one file per VPC.")
rootCmd.PersistentFlags().StringVarP(&args.prefix, prefixFlag, "p", "", "The prefix of the files that will be created.")
rootCmd.PersistentFlags().BoolVarP(&args.locals, localsFlag, "l", false,
"whether to generate a locals.tf file (only possible when the output format is tf)")
rootCmd.PersistentFlags().SortFlags = false

// flags set for all commands
rootCmd.PersistentFlags().SortFlags = false
_ = rootCmd.MarkPersistentFlagRequired(configFlag)
rootCmd.MarkFlagsMutuallyExclusive(outputFileFlag, outputDirFlag)

// sub cmds
rootCmd.AddCommand(NewSynthCommand(args))
rootCmd.AddCommand(NewOptimizeCommand(args))

// disable help command. should use --help flag instead
zivnevo marked this conversation as resolved.
Show resolved Hide resolved
rootCmd.CompletionOptions.HiddenDefaultCmd = true
rootCmd.SetHelpCommand(&cobra.Command{Hidden: true}) // disable help command. should use --help flag instead
rootCmd.SetHelpCommand(&cobra.Command{Hidden: true})

return rootCmd
}
Expand Down
11 changes: 10 additions & 1 deletion cmd/subcmds/synth.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ import (
"github.com/np-guard/vpc-network-config-synthesis/pkg/utils"
)

const specFlag = "spec"

func NewSynthCommand(args *inArgs) *cobra.Command {
cmd := &cobra.Command{
Use: "synth",
Expand All @@ -21,9 +23,16 @@ func NewSynthCommand(args *inArgs) *cobra.Command {
--config and --spec parameters must be supplied.`,
}

// flags
cmd.PersistentFlags().StringVarP(&args.specFile, specFlag, "s", "", "JSON file containing spec file")
cmd.PersistentFlags().StringVarP(&args.outputDir, outputDirFlag, "d", "",
"Write generated resources to files in the specified directory, one file per VPC.")
cmd.PersistentFlags().StringVarP(&args.prefix, prefixFlag, "p", "", "The prefix of the files that will be created.")

// flags settings
_ = cmd.MarkPersistentFlagRequired(specFlag)

// sub cmds
cmd.AddCommand(NewSynthACLCommand(args))
cmd.AddCommand(NewSynthSGCommand(args))

Expand All @@ -41,5 +50,5 @@ func synthesis(cmd *cobra.Command, args *inArgs, newSynthesizer func(*ir.Spec, b
if err != nil {
return err
}
return writeOutput(args, collection, utils.MapKeys(spec.Defs.ConfigDefs.VPCs))
return writeOutput(args, collection, utils.MapKeys(spec.Defs.ConfigDefs.VPCs), true)
}
7 changes: 7 additions & 0 deletions cmd/subcmds/unmarshal.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,3 +26,10 @@ func unmarshal(args *inArgs) (*ir.Spec, error) {

return model, nil
}

func parseCollection(args *inArgs, isSG bool) (ir.Collection, error) {
if isSG {
return confio.ReadSGs(args.configFile)
}
return confio.ReadACLs(args.configFile)
}
24 changes: 24 additions & 0 deletions cmd/subcmds/validateFlags.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
/*
Copyright 2023- IBM Inc. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package subcmds

import "fmt"

func validateFlags(args *inArgs) error {
if args.outputDir != "" && args.outputFile != "" {
return fmt.Errorf("specifying both -d and -o is not allowed")
}
if err := updateOutputFormat(args); err != nil {
return err
}
if args.outputDir != "" && args.outputFmt == jsonOutputFormat {
return fmt.Errorf("-d cannot be used with format json")
}
if args.locals && args.outputFmt != tfOutputFormat {
return fmt.Errorf("--locals flag requires setting the output format to tf")
}
return nil
}
12 changes: 12 additions & 0 deletions pkg/io/confio/parse_acls.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
/*
Copyright 2023- IBM Inc. All Rights Reserved.
SPDX-License-Identifier: Apache-2.0
*/

package confio

import "github.com/np-guard/vpc-network-config-synthesis/pkg/ir"

func ReadACLs(_ string) (*ir.ACLCollection, error) {
return nil, nil
}
Loading
Loading