From 9c0a853a3d3328c63336c6f62cde8f3b6404da47 Mon Sep 17 00:00:00 2001 From: Jude N Date: Wed, 31 Oct 2018 21:58:24 -0400 Subject: [PATCH] YAR SHE BLOWS --- .gitignore | 6 + MANIFEST.in | 5 + README | 58 +++++++++ bin/salty-linter | 73 ++++++++++++ example_sls/requires.sls | 14 +++ example_sls/two-items.sls | 16 +++ requirements.txt | 3 + salty_linter/__init__.py | 139 ++++++++++++++++++++++ salty_linter/data/salty_linter.yaml | 172 +++++++++++++++++++++++++++ salty_linter/data/salty_linter.yaml~ | 66 ++++++++++ setup.py | 28 +++++ test_requirements.txt | 1 + 12 files changed, 581 insertions(+) create mode 100644 .gitignore create mode 100644 MANIFEST.in create mode 100644 README create mode 100755 bin/salty-linter create mode 100644 example_sls/requires.sls create mode 100644 example_sls/two-items.sls create mode 100644 requirements.txt create mode 100644 salty_linter/__init__.py create mode 100644 salty_linter/data/salty_linter.yaml create mode 100644 salty_linter/data/salty_linter.yaml~ create mode 100644 setup.py create mode 100644 test_requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ce0ad42 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +build +dist +env +MANIFEST +*.egg-info +*~ diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..55aafb9 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,5 @@ +include README +include requirements.txt +recursive-include salty_linter * +recursive-include bin * +recursive-include data * diff --git a/README b/README new file mode 100644 index 0000000..6958949 --- /dev/null +++ b/README @@ -0,0 +1,58 @@ +salty_linter +Yar. Matey, A quick and dirty linter for salt. + +TODO: +- some parameters take either a string or a list + +blah: + ,: + - : value + - : + +blah: + : + - + - : value + - : + +- Stage 1: + - Validate common salt-state / warn if an unexpected one is seen + +- Stage 2: + - Validate state/function combo / warn on unexpected combos + - Complain about duplicate state/function combos. + - Complain about mutually exclusive fields (for example 'contents' and 'source' from file.manage) + +- Stage 3: + - Validate function parameter names + - Include global parameters like require, unless etc (https://docs.saltstack.com/en/latest/ref/states/requisites.html) + - Report on missing mandatory parameters (though salt will likely error on this already) + +- Stage 4: + - Validate function argument types + - How to handle 'string or list' type args ? + - Complain about duplicate parameters + - Complain about unexpected parameters + +- Stage 5: + - Handle includes (wrt duplicate state/functions) + +- Stage 6: + - Handle excludes + +- Stage 7: (???) + - Validate common module.run . combos + +- Stage 8: (???) + - Validate common module.run parameters + + +===================================== +Development Notes +> virtualenv --python=python3.5 env +> source ./env/bin/activate + +Build the tar.gz +> python setup.py sdist + + diff --git a/bin/salty-linter b/bin/salty-linter new file mode 100755 index 0000000..c06aa1f --- /dev/null +++ b/bin/salty-linter @@ -0,0 +1,73 @@ +#!/usr/bin/env python + +import argparse +from os import access, R_OK +import os.path +import pkg_resources +import pprint +import sys +import yaml +from ruamel.yaml import YAML + +import salty_linter + +default_types = pkg_resources.resource_filename('salty_linter', 'data/salty_linter.yaml') + +pp = pprint.PrettyPrinter(indent=4) + +# - option to pick an alternate type database, say for a specific version of salt (defaults to salty-linter.yaml) +argparser = argparse.ArgumentParser(description="salty-linter. Let's lint some salt, mateys", + epilog='Now let\'s get some salt states in ship shop shape !') +argparser.add_argument('--verbose', dest='verbose', action='store_true', help="Let's be super chatty and generate a lot of output.") +argparser.add_argument('--no-werror', dest='werror', action='store_false', help="Don't treat warnings as errors") +argparser.add_argument('--typedb', nargs='?', default=default_types, type=argparse.FileType('r'), help="alternative type database.") +argparser.add_argument('sls', nargs='+', help="list of salt state files") +argparser.set_defaults(verbose=False, werror=True) +args = argparser.parse_args() + +pp.pprint(args) + +if args.verbose: + print("HI I'M SALTY_LINTER. LETS GET CHATTY.") + +# Open the type database file and read in the type database +# Error out of that file doesn't parse / doesn't exist / is unreadab +typedb = yaml.load(args.typedb) + +if args.verbose: + print("HERE'S YOUR TYPE DATABSE. WALLOW IN IT.") + pp.pprint(typedb) + print("WRAP UP YOUR TYPE DATABASE WALLOWING.") + +exit_status = 0 +for an_sls_filename in args.sls: + print('PROCESSING: ' + an_sls_filename) + + # Check if the file exists and is readable + fullpath_sls_filename = os.path.abspath(an_sls_filename) + if not os.path.isfile(fullpath_sls_filename): + print("error: {} is not a file".format(an_sls_filename)) + exit_status = 1 + continue + elif not access(fullpath_sls_filename, R_OK): + print("error: {} is not readable".format(an_sls_filename)) + exit_status = 1 + continue + + with open(fullpath_sls_filename) as fp: + yaml = YAML(typ='rt') + sls_yaml = yaml.load(fp) + (lint_errors, lint_warnings) = salty_linter.lint(sls_yaml, typedb) + if lint_errors or (lint_warnings and args.werror): + exit_status = 1 + for (line_number, an_error) in lint_errors: + print("Error: " + an_sls_filename + ": " + str(line_number) + ": " + an_error) + for (line_number, a_warning) in lint_warnings: + print("Warning: " + an_sls_filename + ": " + str(line_number) + ": " + a_warning) + +if exit_status == 0: + print("NO OBVIOUS FAILURES.") +else: + print("MORE SHIP-SHOPPING FOR YOU !") +sys.exit(exit_status) + diff --git a/example_sls/requires.sls b/example_sls/requires.sls new file mode 100644 index 0000000..5c69090 --- /dev/null +++ b/example_sls/requires.sls @@ -0,0 +1,14 @@ +# I WROTE THIS LINTER AFTER RUNNING INTO THIS PROBLEM MULTIPLE TIME + +require_not_requires: + file.managed: + - contents: "it's require not requires" + - requires: + - cmd: "/bin/echo OOPS I DID IT AGAIN" + + +multiball: + pkg: + - installed + service: + - running diff --git a/example_sls/two-items.sls b/example_sls/two-items.sls new file mode 100644 index 0000000..5584dd1 --- /dev/null +++ b/example_sls/two-items.sls @@ -0,0 +1,16 @@ +# This file has two states + +/etc/motd: + file.managed: + - contents: "I yam who I yam" + - user: poppey + - group: sailors + +# Also this state has no function parameters +/etc/nope: + file.missing + +# This state has no parameters but a bad function name +/etc/nopesir: + file.awol + diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8bfbd06 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +pyyaml +ruamel.yaml +ruamel.yaml[jinja2] diff --git a/salty_linter/__init__.py b/salty_linter/__init__.py new file mode 100644 index 0000000..a5cf233 --- /dev/null +++ b/salty_linter/__init__.py @@ -0,0 +1,139 @@ +import pprint + +def lint(sls_yaml, types): + """ + Check the sls_yaml for Salt errors / warnings + + typedb : Nested dictionary of Salt states/functions/parameters (partial) + sls_yaml: ruamel.yaml[jinja2].load()'d output for some Salt state file. + + Returns: (errors, warnings) + errors: List of (line#, error) tuples + warnings: List of (line#, warnings) tuples + """ + + pp = pprint.PrettyPrinter(indent=4) + #pp.pprint(dir(sls_yaml)) + #pp.pprint(sls_yaml.__dict__) + + + def lint_parameters(function_name, parameters, parameter_types): + """ + Input: + function_name: String of the form "." currently being linted + parameters: Map of {paramter_name: parameter_data} items currently being linted + parameter_types: Map of {parameter_name: parameter_type} items + Output: + errors: List of error string + warnings: List of warning strings + """ + errors = [] + warnings = [] + for parameter in parameters: + parameter_name = next(iter(parameter.keys())) + + ## parameter is either just a string or a {'parameter' : data} map, or always a map ? + + if parameter_name in types['globals']: + continue + elif parameter_name not in parameter_types: + s_warning = "Unexpected parameter name {} for {}" + warnings.append((parameter.lc.line + 1, s_warning.format(parameter_name, function_name))) + else: + ## TODO: Add that parameter.values()[0] matches the against parameter_types[parameter] type + pass + return (errors, warnings) + + + def lint_function(state_name, function_calls, function_types): + """ + Input: + state_name: String of the state_name currently being linted + function_calls: Map of {function_name: parameters} items currently being linteds + function_types; Map of {function_name: parameter_types} items + Output: + errors: List of error string + warnings: List of warning strings + """ + errors = [] + warnings = [] + for function_name, parameters in function_calls.items(): + if function_name not in function_types: + warnings.append((parameters.lc.line+1, "Unexpected function name {}.{}.".format(state_name, function_name))) + continue + else: + (parameter_errors, parameter_warnings) = lint_parameters(state_name + "." + function_name, parameters, function_types[function_name]) + errors += parameter_errors + warnings += parameter_warnings + return (errors, warnings) + + ## ------------------------------ + + state_types = types['states'] + + errors = [] + warnings = [] + for label, label_value in sls_yaml.items(): + + # It's possible the label_value is just a 'state.function'. See example_sls/two-items.sls + if not isinstance(label_value, dict): + value = {label_value: []} + else: + value = label_value + + for state_function_name, function_calls_or_parameters in value.items(): + + # state_function_name should be 'state_name' or 'state_name.function_name' + # sosf => 'state or state.function' + # = [state_name] or [state_name, function_name] + sosf = state_function_name.split('.') + + # Verify that sosf[0] is an expected state: + state_name = '' + if sosf[0] not in state_types: + warnings.append((value.lc.line + 1, "Unexpected state name {}.".format(sosf[0]))) + continue + else: + state_name = sosf[0] + + # "state" case + # function_calls_or_parameters[0] will be the function name + # function_call_or_parameters[1:] will be the parameters + if len(sosf) == 1: + + if not function_calls_or_parameters: + errors.append(label_value.lc.line + 1, "Missing function name in parameter list for state {}.".format(state_name)) + continue + + function_name = function_calls_or_parameters[0] + if function_name not in state_types[state_name]: + warnings.append((state_function_name.lc.line + 1, "Unexpected function name {} for {}.".format(function_name, state_name))) + continue + parameters = function_call_or_parameters[1:] + (function_errors, function_warnings) = lint_function(state_name + "." + function_name, parameters, state_types[state_name][function_name]) + errors += function_errors + warnings += function_warnings + + # "state.function" case + # function_calls_or_parameters will be a list of parameters + elif len(sosf) == 2: + function_name = sosf[1] + if function_name not in state_types[state_name]: + + # TODO: This gives the line number at the top of the label block which isn't great. + # It would be better to give the exact like number of the invalid function + warnings.append((sls_yaml.lc.key(label)[0] + 1, "Unexpected function name {} for {}.".format(function_name, state_name))) + continue + + # Children should be parameters of "state.function" + (parameter_errors, parameter_warnings) = lint_parameters(state_name + "." + function_name, function_calls_or_parameters, state_types[state_name][function_name]) + errors += parameter_errors + warnings += parameter_warnings + + # Anything other than "state" or "state.function" is an error. + else: + errors.append((value.lc.line + 1, "Expected or .: Too many '.'s in {}".format(state_function_name))) + continue + + return(errors, warnings) + diff --git a/salty_linter/data/salty_linter.yaml b/salty_linter/data/salty_linter.yaml new file mode 100644 index 0000000..d5a51f3 --- /dev/null +++ b/salty_linter/data/salty_linter.yaml @@ -0,0 +1,172 @@ +--- + +# Based on the 2018.3.2 salt release +# This is a partial state/function/parameter database based on the states I use +# Please submit merge requests for additional coverage + +type: + basic: + - boolean + - char + - dict + - list + - string + - solos # string_or_list_of_strings: String | [String] + template: + - cheetah + - genshi + - jinja + - mako + - py + - wempy + +# global state arguments +globals: + - check_cmd + - fire_event + - listen + - listen_in + - mod_run_check + - mod_run_check_cmd + - onchanges + - onchanges_any + - onchanges_in + - onfail + - onfail_any + - onfail_in + - onlyif + - prereq + - prereq_in + - require + - require_in + - require_any + - reload_grains + - reload_modules + - reload_pillar + - retry # additional parameter checking available here (attempts, until, interval, splay) + - runas + - unas_password + - unless + - use + - use_in + - watch + - watch_any + - watch_in + +# states: +# state-name: +# function-name: +# parameter: type +# ... +# ... +# ... +states: + file: + directory: + allow_symlink: boolean + backupname: string + children_only: boolean + clean: integer + dir_mode: string + exclude_pat: boolean + file_mode: string + follow_symlinks; boolean + force: boolean + group: string + makedirs: boolean + max_depth: integer + mode: string + name: string + recurse: list + user: string + win_owner: string + win_perms: dict + win_deny_perms: dict + win_inheritence: boolean + win_perms_reset: boolean + managed: + allow_empty: boolean + attrs: string + backup: string + check_cmd: string + contents: solos + contents_delimiter: char + contents_grains: string + contents_newline: boolean + contents_pillar: string + context: dict + create: boolean + defaults: dict + dir_mode: string + encoding: stirng + encoding_errors: string + follow_symlinks: boolean + group: string + keep_source: boolean + makedirs: boolean + mode: string + name: string + replace: boolean + skip_verify: boolean + show_changes: boolean + source: solos + source_hash: string + source_hash_name: string + template: template + tmp_ext: string + user: string + win_deny_perms: dict + win_inheritance: boolean + win_owner: string + win_perms: dict + win_perms_reset: boolean + missing: + name: string + touch: + atime: string + mtime: string + makedirs: boolean + name: string + + pkg: + installed: + allow_updates: boolean + cache_valid_time: string + fromrepo: string + hold: boolean + ignore_epoch: boolean + ignore_types: list + install_recommends: boolean + name: string + names: list + normalize: bool + only_upgrade: boolean + pkg_verify: bool + pkgs: list + refresh: boolean + report_reboot_exit_codes: boolean + resolve_capabilities: boolean + skip_suggestions: boolean + skip_verify: boolean + sources: list + update_holds: boolean + verify_options: list + version: string + + selinux: + mode: + name: string + + service: + enabled: + name: string + running: + enable: boolean + init_delay: integer + name: string + no_block: boolean + sig: string + unmask: boolean + unmask_runtime: boolean + + diff --git a/salty_linter/data/salty_linter.yaml~ b/salty_linter/data/salty_linter.yaml~ new file mode 100644 index 0000000..368dab8 --- /dev/null +++ b/salty_linter/data/salty_linter.yaml~ @@ -0,0 +1,66 @@ +--- + +# Based on the 2018.3.2 salt release +# This is a partial state/function/parameter database based on the states I use +# Please submit merge requests for additional coverage + +type: + basic: + - boolean + - char + - mos # map of strings + - string + - solos # string_or_list_of_strings: String | [String] + template: + - cheetah + - genshi + - jinja + - mako + - py + - wempy + +# states: +# state-name: +# function-name: +# parameter: type +# ... +# ... +# ... +states: + file: + managed: + allow_empty: boolean + attrs: string + backup: string + check_cmd: string + contents: solos + contents_delimiter: char + contents_grains: string + contents_newline: boolean + contents_pillar: string + context: mos + create: boolean + defaults: mos + dir_mode: string + encoding: stirng + encoding_errors: string + follow_symlinks: boolean + group: string + keep_source: boolean + makedirs: boolean + mode: string + name: string + replace: boolean + skip_verify: boolean + show_changes: boolean + source: solos + source_hash: string + source_hash_name: string + template: template + tmp_ext: string + user: string + win_deny_perms: mos + win_inheritance: boolean + win_owner: string + win_perms: mos + win_perms_reset: boolean diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..07c5e13 --- /dev/null +++ b/setup.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python + +from distutils.core import setup +import setuptools + +with open('requirements.txt') as fp: + install_requires = fp.read() + +with open('test_requirements.txt') as fp: + tests_require = fp.read() + +setup( + name = 'salty_linter', + version = '1.0.0', + author = 'Jude N', + author_email = 'its_still_require_not_requires_matey@pwan.org', + description = 'A salt linter', + long_description = 'A salt linter that checks for common salt state/function/parameters combos', + url = "https://pwan.org/git/?p=salty_linter;a=summary", + packages = ['salty_linter'], + package_dir = {'salty_linter' : 'salty_linter'}, + + install_requires = install_requires, + tests_require = tests_require, + + package_data = {'salty_linter': ['data/*.yaml']}, + scripts = ['bin/salty-linter'] +) diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 0000000..e079f8a --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1 @@ +pytest -- 2.39.2