remove AWS docs from wheel and minify JSON

Author: gruebel

.pre-commit-config.yaml

@@ -13,9 +13,9 @@ repos:
     rev: v0.5.3
     hooks:
       - id: ruff
-        files: ^policy_sentry/
+        files: ^(policy_sentry/|setup.py)
       - id: ruff-format
-        files: ^policy_sentry/
+        files: ^(policy_sentry/|setup.py)
   - repo: https://github.com/Lucas-C/pre-commit-hooks-safety
     rev: v1.3.3
     hooks:

MANIFEST.in

@@ -6,7 +6,6 @@
 from __future__ import annotations
 
 import logging
-import os
 import shutil
 from pathlib import Path
 
@@ -17,7 +16,6 @@
 from policy_sentry.shared.constants import (
     BUNDLED_DATA_DIRECTORY,
     BUNDLED_DATASTORE_FILE_PATH,
-    BUNDLED_HTML_DIRECTORY_PATH,
     CONFIG_DIRECTORY,
     DATASTORE_FILE_PATH,
     LOCAL_ACCESS_OVERRIDES_FILE,
@@ -101,8 +99,8 @@ def initialize(
     # Create the config directory
     database_path = create_policy_sentry_config_directory()
 
-    # Copy over the html docs, which will be used to build the database
-    create_html_docs_directory()
+    # Create local html docs folder, if it doesn't exist
+    LOCAL_HTML_DIRECTORY_PATH.mkdir(parents=True, exist_ok=True)
 
     # Create overrides file, which allows us to override the Access Levels
     # provided by AWS documentation
@@ -126,6 +124,9 @@ def initialize(
     if fetch:
         # `wget` the html docs to the local directory
         update_html_docs_directory(LOCAL_HTML_DIRECTORY_PATH)
+    elif not next(LOCAL_HTML_DIRECTORY_PATH.glob("*.html"), None):
+        print("No HTML docs found, fetching from AWS!")
+        update_html_docs_directory(LOCAL_HTML_DIRECTORY_PATH)
 
     # --build
     if build or access_level_overrides_file or fetch:
@@ -134,9 +135,8 @@ def initialize(
 
     # Query the database for all the services that are now in the database.
     all_aws_service_prefixes = get_all_service_prefixes()
-    total_count_of_services = str(len(all_aws_service_prefixes))
     print("Initialization complete!")
-    print(f"Total AWS services in the IAM database: {total_count_of_services}")
+    print(f"Total AWS services in the IAM database: {len(all_aws_service_prefixes)}")
     logger.debug("\nService prefixes:")
     logger.debug(", ".join(all_aws_service_prefixes))
 
@@ -157,22 +157,3 @@ def create_policy_sentry_config_directory() -> Path:
     else:
         CONFIG_DIRECTORY.mkdir(exist_ok=True)
     return LOCAL_DATASTORE_FILE_PATH
-
-
-def create_html_docs_directory() -> None:
-    """
-    Copies the HTML files from the pip package over to its own folder in the CONFIG_DIRECTORY.
-    Essentially:
-    mkdir -p ~/.policy_sentry/data/docs
-    cp -r $MODULE_DIR/policy_sentry/shared/data/docs ~/.policy_sentry/data/docs
-    :return:
-    """
-    if os.path.exists(LOCAL_HTML_DIRECTORY_PATH):
-        pass
-    else:
-        os.makedirs(LOCAL_HTML_DIRECTORY_PATH)
-    # Copy from the existing html docs folder - the path ./policy_sentry/shared/data/docs within this repository
-    logger.debug(BUNDLED_HTML_DIRECTORY_PATH)
-    if os.path.exists(LOCAL_HTML_DIRECTORY_PATH):
-        shutil.rmtree(LOCAL_HTML_DIRECTORY_PATH)
-    shutil.copytree(BUNDLED_HTML_DIRECTORY_PATH, LOCAL_HTML_DIRECTORY_PATH)

policy_sentry/command/initialize.py

@@ -1,10 +1,13 @@
 """Setup script for Policy Sentry"""
 
-import setuptools
-import os
+import json
 import re
+from pathlib import Path
+
+import setuptools
+from setuptools.command.build_py import build_py
 
-HERE = os.path.abspath(os.path.dirname(__file__))
+HERE = Path(__file__).parent
 VERSION_RE = re.compile(r"""__version__ = ['"]([0-9.]+)['"]""")
 TESTS_REQUIRE = ["coverage", "pytest"]
 REQUIRED_PACKAGES = [
@@ -23,15 +26,32 @@
 }
 
 
-def get_version():
-    init = open(os.path.join(HERE, "policy_sentry", "bin", "version.py")).read()
+class PreBuildCommand(build_py):
+    """Pre-build command"""
+
+    def minify_iam_data_json(self) -> None:
+        """Minifies the IAM DB JSON file"""
+        src_iam_data_path = Path("policy_sentry/shared/data/iam-definition.json")
+        build_iam_data_path = Path(self.build_lib) / src_iam_data_path
+
+        self.mkpath(str(build_iam_data_path.parent))
+        minified = json.dumps(
+            json.loads(src_iam_data_path.read_bytes()), separators=(",", ":")
+        )
+        build_iam_data_path.write_text(minified)
+
+    def run(self) -> None:
+        self.execute(self.minify_iam_data_json, ())
+        build_py.run(self)
+
+
+def get_version() -> str:
+    init = (HERE / "policy_sentry/bin/version.py").read_text()
     return VERSION_RE.search(init).group(1)
 
 
-def get_description():
-    return open(
-        os.path.join(os.path.abspath(HERE), "README.md"), encoding="utf-8"
-    ).read()
+def get_description() -> str:
+    (HERE / "README.md").read_text()
 
 
 setuptools.setup(
@@ -45,6 +65,14 @@ def get_description():
     long_description_content_type="text/markdown",
     url="https://github.com/salesforce/policy_sentry",
     packages=setuptools.find_packages(exclude=["test*"]),
+    package_data={
+        "policy_sentry": ["py.typed"],
+        "policy_sentry.shared": [
+            "data/*.json",
+            "data/*.yml",
+            "data/audit/*.txt",
+        ],
+    },
     tests_require=TESTS_REQUIRE,
     install_requires=REQUIRED_PACKAGES,
     project_urls=PROJECT_URLS,
@@ -64,4 +92,7 @@ def get_description():
     zip_safe=True,
     keywords="aws iam roles policy policies privileges security",
     python_requires=">=3.8",
+    cmdclass={
+        "build_py": PreBuildCommand,
+    },
 )

setup.py

@@ -1,9 +1,12 @@
 #!/usr/bin/env python
+import shutil
 import sys
 import os
 import logging
 from invoke import task, Collection, UnexpectedExit, Failure
 
+from policy_sentry.shared.constants import LOCAL_HTML_DIRECTORY_PATH, BUNDLED_HTML_DIRECTORY_PATH
+
 sys.path.append(
     os.path.abspath(
         os.path.join(os.path.dirname(__file__), os.path.pardir, "policy_sentry")
@@ -63,7 +66,7 @@ def build_package(c):
 @task(pre=[build_package])
 def install_package(c):
     """Install the policy_sentry package built from the current directory contents (not PyPi)"""
-    c.run("pip3 install -q dist/policy_sentry-*.tar.gz")
+    c.run("pip3 install -q dist/policy_sentry-*.whl")
 
 
 @task
@@ -110,6 +113,9 @@ def clean_config_directory(c):
 def create_db(c):
     """Integration testing: Initialize the policy_sentry database"""
     try:
+        LOCAL_HTML_DIRECTORY_PATH.mkdir(parents=True, exist_ok=True)
+        shutil.copytree(BUNDLED_HTML_DIRECTORY_PATH, LOCAL_HTML_DIRECTORY_PATH, dirs_exist_ok=True)
+
         initialize.initialize("")
     except UnexpectedExit as u_e:
         logger.critical(f"FAIL! UnexpectedExit: {u_e}")