Merge pull request #20 from astrofrog/embed_python_script

Embed python script in YAML file

Author: ConorMacBride

.github/workflows/publish.yml

@@ -60,15 +60,15 @@ jobs:
       matrix: ${{ steps.set-outputs.outputs.matrix }}
       upload_to_pypi: ${{ steps.set-upload.outputs.upload_to_pypi }}
     steps:
-      - uses: actions/checkout@v2
-        with:
-           repository: 'OpenAstronomy/github-actions-workflows'
       - uses: actions/setup-python@v2
         with:
           python-version: '3.9'
       - run: python -m pip install PyYAML click
+      - run: echo $LOAD_BUILD_TARGETS_SCRIPT | base64 --decode > load_build_targets.py
+        env:
+          LOAD_BUILD_TARGETS_SCRIPT: aW1wb3J0IGpzb24KaW1wb3J0IG9zCgppbXBvcnQgY2xpY2sKaW1wb3J0IHlhbWwKCk1BQ0hJTkVfVFlQRSA9IHsKICAgICJsaW51eCI6ICJ1YnVudHUtbGF0ZXN0IiwKICAgICJtYWNvcyI6ICJtYWNvcy1sYXRlc3QiLAogICAgIndpbmRvd3MiOiAid2luZG93cy1sYXRlc3QiLAp9CgpDSUJXX0JVSUxEID0gb3MuZW52aXJvbi5nZXQoIkNJQldfQlVJTEQiLCAiKiIpCkNJQldfQVJDSFMgPSBvcy5lbnZpcm9uLmdldCgiQ0lCV19BUkNIUyIsICJhdXRvIikKCgpAY2xpY2suY29tbWFuZCgpCkBjbGljay5vcHRpb24oIi0tdGFyZ2V0cyIsIGRlZmF1bHQ9IiIpCmRlZiBsb2FkX2J1aWxkX3RhcmdldHModGFyZ2V0cyk6CiAgICAiIiJTY3JpcHQgdG8gbG9hZCBjaWJ1aWxkd2hlZWwgdGFyZ2V0cyBmb3IgR2l0SHViIEFjdGlvbnMgd29ya2Zsb3cuIiIiCiAgICAjIExvYWQgbGlzdCBvZiB0YXJnZXRzCiAgICB0YXJnZXRzID0geWFtbC5sb2FkKHRhcmdldHMsIExvYWRlcj15YW1sLkJhc2VMb2FkZXIpCiAgICBwcmludChqc29uLmR1bXBzKHRhcmdldHMsIGluZGVudD0yKSkKCiAgICAjIENyZWF0ZSBtYXRyaXgKICAgIG1hdHJpeCA9IHsiaW5jbHVkZSI6IFtdfQogICAgZm9yIHRhcmdldCBpbiB0YXJnZXRzOgogICAgICAgIG1hdHJpeFsiaW5jbHVkZSJdLmFwcGVuZChnZXRfbWF0cml4X2l0ZW0odGFyZ2V0KSkKCiAgICAjIE91dHB1dCBtYXRyaXgKICAgIHByaW50KGpzb24uZHVtcHMobWF0cml4LCBpbmRlbnQ9MikpCiAgICBwcmludChmIjo6c2V0LW91dHB1dCBuYW1lPW1hdHJpeDo6e2pzb24uZHVtcHMobWF0cml4KX0iKQoKCmRlZiBnZXRfb3ModGFyZ2V0KToKICAgIGlmICJtYWNvcyIgaW4gdGFyZ2V0OgogICAgICAgIHJldHVybiBNQUNISU5FX1RZUEVbIm1hY29zIl0KICAgIGlmICJ3aW4iIGluIHRhcmdldDoKICAgICAgICByZXR1cm4gTUFDSElORV9UWVBFWyJ3aW5kb3dzIl0KICAgIHJldHVybiBNQUNISU5FX1RZUEVbImxpbnV4Il0KCgpkZWYgZ2V0X2NpYndfYnVpbGQodGFyZ2V0KToKICAgIGlmIHRhcmdldCBpbiB7ImxpbnV4IiwgIm1hY29zIiwgIndpbmRvd3MifToKICAgICAgICByZXR1cm4gQ0lCV19CVUlMRAogICAgcmV0dXJuIHRhcmdldAoKCmRlZiBnZXRfY2lid19hcmNocyh0YXJnZXQpOgogICAgZm9yIGFyY2ggaW4gWyJhYXJjaDY0IiwgImFybTY0IiwgInVuaXZlcnNhbDIiXToKICAgICAgICBpZiB0YXJnZXQuZW5kc3dpdGgoYXJjaCk6CiAgICAgICAgICAgIHJldHVybiBhcmNoCiAgICByZXR1cm4gQ0lCV19BUkNIUwoKCmRlZiBnZXRfbWF0cml4X2l0ZW0odGFyZ2V0KToKICAgIHJldHVybiB7CiAgICAgICAgInRhcmdldCI6IHRhcmdldCwKICAgICAgICAib3MiOiBnZXRfb3ModGFyZ2V0KSwKICAgICAgICAiQ0lCV19CVUlMRCI6IGdldF9jaWJ3X2J1aWxkKHRhcmdldCksCiAgICAgICAgIkNJQldfQVJDSFMiOiBnZXRfY2lid19hcmNocyh0YXJnZXQpLAogICAgfQoKCmlmIF9fbmFtZV9fID09ICJfX21haW5fXyI6CiAgICBsb2FkX2J1aWxkX3RhcmdldHMoKQo=
       - id: set-outputs
-        run: python tools/load_build_targets.py --targets "${{ inputs.targets }}"
+        run: python load_build_targets.py --targets "${{ inputs.targets }}"
         shell: sh
       - id: set-upload
         run: |

.github/workflows/tox.yml

@@ -75,16 +75,17 @@ jobs:
     outputs:
       matrix: ${{ steps.set-outputs.outputs.matrix }}
     steps:
-      - uses: actions/checkout@v2
-        with:
-          repository: 'OpenAstronomy/github-actions-workflows'
       - uses: actions/setup-python@v2
         with:
           python-version: '3.9'
       - run: python -m pip install PyYAML click
+      - run: echo $TOX_MATRIX_SCRIPT | base64 --decode > tox_matrix.py
+        env:
+          TOX_MATRIX_SCRIPT: aW1wb3J0IGpzb24KaW1wb3J0IHJlCgppbXBvcnQgY2xpY2sKaW1wb3J0IHlhbWwKCgpAY2xpY2suY29tbWFuZCgpCkBjbGljay5vcHRpb24oIi0tZW52cyIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tbGlicmFyaWVzIiwgZGVmYXVsdD0iIikKQGNsaWNrLm9wdGlvbigiLS1wb3NhcmdzIiwgZGVmYXVsdD0iIikKQGNsaWNrLm9wdGlvbigiLS10b3hkZXBzIiwgZGVmYXVsdD0iIikKQGNsaWNrLm9wdGlvbigiLS10b3hhcmdzIiwgZGVmYXVsdD0iIikKQGNsaWNrLm9wdGlvbigiLS1weXRlc3QiLCBkZWZhdWx0PSJ0cnVlIikKQGNsaWNrLm9wdGlvbigiLS1jb3ZlcmFnZSIsIGRlZmF1bHQ9IiIpCkBjbGljay5vcHRpb24oIi0tY29uZGEiLCBkZWZhdWx0PSJhdXRvIikKQGNsaWNrLm9wdGlvbigiLS1kaXNwbGF5IiwgZGVmYXVsdD0iZmFsc2UiKQpAY2xpY2sub3B0aW9uKCItLXJ1bnMtb24iLCBkZWZhdWx0PSIiKQpAY2xpY2sub3B0aW9uKCItLWRlZmF1bHQtcHl0aG9uIiwgZGVmYXVsdD0iIikKZGVmIGxvYWRfdG94X3RhcmdldHMoZW52cywgbGlicmFyaWVzLCBwb3NhcmdzLCB0b3hkZXBzLCB0b3hhcmdzLCBweXRlc3QsCiAgICAgICAgICAgICAgICAgICAgIGNvdmVyYWdlLCBjb25kYSwgZGlzcGxheSwgcnVuc19vbiwgZGVmYXVsdF9weXRob24pOgogICAgIiIiU2NyaXB0IHRvIGxvYWQgdG94IHRhcmdldHMgZm9yIEdpdEh1YiBBY3Rpb25zIHdvcmtmbG93LiIiIgogICAgIyBMb2FkIGVudnMgY29uZmlnCiAgICBlbnZzID0geWFtbC5sb2FkKGVudnMsIExvYWRlcj15YW1sLkJhc2VMb2FkZXIpCiAgICBwcmludChqc29uLmR1bXBzKGVudnMsIGluZGVudD0yKSkKCiAgICAjIExvYWQgZ2xvYmFsIGxpYnJhcmllcyBjb25maWcKICAgIGdsb2JhbF9saWJyYXJpZXMgPSB7CiAgICAgICAgImJyZXciOiBbXSwKICAgICAgICAiYnJldy1jYXNrIjogW10sCiAgICAgICAgImFwdCI6IFtdLAogICAgICAgICJjaG9jbyI6IFtdLAogICAgfQogICAgbGlicmFyaWVzID0geWFtbC5sb2FkKGxpYnJhcmllcywgTG9hZGVyPXlhbWwuQmFzZUxvYWRlcikKICAgIGlmIGxpYnJhcmllcyBpcyBub3QgTm9uZToKICAgICAgICBnbG9iYWxfbGlicmFyaWVzLnVwZGF0ZShsaWJyYXJpZXMpCiAgICBwcmludChqc29uLmR1bXBzKGdsb2JhbF9saWJyYXJpZXMsIGluZGVudD0yKSkKCiAgICAjIERlZmF1bHQgaW1hZ2VzIHRvIHVzZSBmb3IgcnVubmVycwogICAgZGVmYXVsdF9ydW5zX29uID0gewogICAgICAgICJsaW51eCI6ICJ1YnVudHUtbGF0ZXN0IiwKICAgICAgICAibWFjb3MiOiAibWFjb3MtbGF0ZXN0IiwKICAgICAgICAid2luZG93cyI6ICJ3aW5kb3dzLWxhdGVzdCIsCiAgICB9CiAgICBjdXN0b21fcnVuc19vbiA9IHlhbWwubG9hZChydW5zX29uLCBMb2FkZXI9eWFtbC5CYXNlTG9hZGVyKQogICAgaWYgaXNpbnN0YW5jZShjdXN0b21fcnVuc19vbiwgZGljdCk6CiAgICAgICAgZGVmYXVsdF9ydW5zX29uLnVwZGF0ZShjdXN0b21fcnVuc19vbikKICAgIHByaW50KGpzb24uZHVtcHMoZGVmYXVsdF9ydW5zX29uLCBpbmRlbnQ9MikpCgogICAgIyBEZWZhdWx0IHN0cmluZyBwYXJhbWV0ZXJzIHdoaWNoIGNhbiBiZSBvdmVyd3JpdHRlbiBieSBlYWNoIGVudgogICAgc3RyaW5nX3BhcmFtZXRlcnMgPSB7CiAgICAgICAgInBvc2FyZ3MiOiBwb3NhcmdzLAogICAgICAgICJ0b3hkZXBzIjogdG94ZGVwcywKICAgICAgICAidG94YXJncyI6IHRveGFyZ3MsCiAgICAgICAgInB5dGVzdCI6IHB5dGVzdCwKICAgICAgICAiY292ZXJhZ2UiOiBjb3ZlcmFnZSwKICAgICAgICAiY29uZGEiOiBjb25kYSwKICAgICAgICAiZGlzcGxheSI6IGRpc3BsYXksCiAgICB9CgogICAgIyBDcmVhdGUgbWF0cml4CiAgICBtYXRyaXggPSB7ImluY2x1ZGUiOiBbXX0KICAgIGZvciBlbnYgaW4gZW52czoKICAgICAgICBtYXRyaXhbImluY2x1ZGUiXS5hcHBlbmQoZ2V0X21hdHJpeF9pdGVtKAogICAgICAgICAgICBlbnYsCiAgICAgICAgICAgIGdsb2JhbF9saWJyYXJpZXM9Z2xvYmFsX2xpYnJhcmllcywKICAgICAgICAgICAgZ2xvYmFsX3N0cmluZ19wYXJhbWV0ZXJzPXN0cmluZ19wYXJhbWV0ZXJzLAogICAgICAgICAgICBydW5zX29uPWRlZmF1bHRfcnVuc19vbiwKICAgICAgICAgICAgZGVmYXVsdF9weXRob249ZGVmYXVsdF9weXRob24sCiAgICAgICAgKSkKCiAgICAjIE91dHB1dCBtYXRyaXgKICAgIHByaW50KGpzb24uZHVtcHMobWF0cml4LCBpbmRlbnQ9MikpCiAgICBwcmludChmIjo6c2V0LW91dHB1dCBuYW1lPW1hdHJpeDo6e2pzb24uZHVtcHMobWF0cml4KX0iKQoKCmRlZiBnZXRfbWF0cml4X2l0ZW0oZW52LCBnbG9iYWxfbGlicmFyaWVzLCBnbG9iYWxfc3RyaW5nX3BhcmFtZXRlcnMsCiAgICAgICAgICAgICAgICAgICAgcnVuc19vbiwgZGVmYXVsdF9weXRob24pOgoKICAgICMgZGVmaW5lIHNwZWMgZm9yIGVhY2ggbWF0cml4IGluY2x1ZGUgKCsgZ2xvYmFsX3N0cmluZ19wYXJhbWV0ZXJzKQogICAgaXRlbSA9IHsKICAgICAgICAib3MiOiBOb25lLAogICAgICAgICJ0b3hlbnYiOiBOb25lLAogICAgICAgICJweXRob25fdmVyc2lvbiI6IE5vbmUsCiAgICAgICAgIm5hbWUiOiBOb25lLAogICAgICAgICJweXRlc3RfZmxhZyI6IE5vbmUsCiAgICAgICAgImxpYnJhcmllc19icmV3IjogTm9uZSwKICAgICAgICAibGlicmFyaWVzX2JyZXdfY2FzayI6IE5vbmUsCiAgICAgICAgImxpYnJhcmllc19hcHQiOiBOb25lLAogICAgICAgICJsaWJyYXJpZXNfY2hvY28iOiBOb25lLAogICAgfQogICAgZm9yIHN0cmluZ19wYXJhbSwgZGVmYXVsdCBpbiBnbG9iYWxfc3RyaW5nX3BhcmFtZXRlcnMuaXRlbXMoKToKICAgICAgICBlbnZfdmFsdWUgPSBlbnYuZ2V0KHN0cmluZ19wYXJhbSkKICAgICAgICBpdGVtW3N0cmluZ19wYXJhbV0gPSBkZWZhdWx0IGlmIGVudl92YWx1ZSBpcyBOb25lIGVsc2UgZW52X3ZhbHVlCgogICAgIyBzZXQgb3MgYW5kIHRveGVudgogICAgZm9yIGssIHYgaW4gcnVuc19vbi5pdGVtcygpOgogICAgICAgIGlmIGsgaW4gZW52OgogICAgICAgICAgICBwbGF0Zm9ybSA9IGsKICAgICAgICAgICAgaXRlbVsib3MiXSA9IGVudi5nZXQoInJ1bnMtb24iLCB2KQogICAgICAgICAgICBpdGVtWyJ0b3hlbnYiXSA9IGVudltrXQogICAgYXNzZXJ0IGl0ZW1bIm9zIl0gaXMgbm90IE5vbmUgYW5kIGl0ZW1bInRveGVudiJdIGlzIG5vdCBOb25lCgogICAgIyBzZXQgcHl0aG9uX3ZlcnNpb24KICAgIG0gPSByZS5zZWFyY2goIl5weSgyfDMpKFswLTldKykiLCBpdGVtWyJ0b3hlbnYiXSkKICAgIGlmIG0gaXMgbm90IE5vbmU6CiAgICAgICAgbWFqb3IsIG1pbm9yID0gbS5ncm91cHMoKQogICAgICAgIGl0ZW1bInB5dGhvbl92ZXJzaW9uIl0gPSBmInttYWpvcn0ue21pbm9yfSIKICAgIGVsc2U6CiAgICAgICAgaXRlbVsicHl0aG9uX3ZlcnNpb24iXSA9IGVudi5nZXQoImRlZmF1bHRfcHl0aG9uIikgb3IgZGVmYXVsdF9weXRob24KCiAgICAjIHNldCBuYW1lCiAgICBpdGVtWyJuYW1lIl0gPSBlbnYuZ2V0KCJuYW1lIikgb3IgaXRlbVsidG94ZW52Il0KCiAgICAjIHNldCBweXRlc3RfZmxhZwogICAgaWYgaXRlbVsicHl0ZXN0Il0gPT0gInRydWUiOgogICAgICAgIHNlcCA9ICJcXCIgaWYgcGxhdGZvcm0gPT0gIndpbmRvd3MiIGVsc2UgIi8iCiAgICAgICAgaXRlbVsicHl0ZXN0X2ZsYWciXSA9ICgKICAgICAgICAgICAgcmYiLS1qdW5pdHhtbD1qdW5pdHtzZXB9dGVzdC1yZXN1bHRzLnhtbCAiCiAgICAgICAgICAgIHJmIi0tY292LXJlcG9ydD14bWw6JHt7R0lUSFVCX1dPUktTUEFDRX19e3NlcH1jb3ZlcmFnZS54bWwiKQogICAgZWxzZToKICAgICAgICBpdGVtWyJweXRlc3RfZmxhZyJdID0gIiIKCiAgICAjIHNldCBsaWJyYXJpZXMKICAgIGVudl9saWJyYXJpZXMgPSBlbnYuZ2V0KCJsaWJyYXJpZXMiKQogICAgaWYgaXNpbnN0YW5jZShlbnZfbGlicmFyaWVzLCBzdHIpIGFuZCBsZW4oZW52X2xpYnJhcmllcy5zdHJpcCgpKSA9PSAwOgogICAgICAgIGVudl9saWJyYXJpZXMgPSB7fSAgIyBubyBsaWJyYXJpZXMgcmVxdWVzdGVkIGZvciBlbnZpcm9ubWVudAogICAgbGlicmFyaWVzID0gZ2xvYmFsX2xpYnJhcmllcyBpZiBlbnZfbGlicmFyaWVzIGlzIE5vbmUgZWxzZSBlbnZfbGlicmFyaWVzCiAgICBmb3IgbWFuYWdlciBpbiBbImJyZXciLCAiYnJld19jYXNrIiwgImFwdCIsICJjaG9jbyJdOgogICAgICAgIGl0ZW1bZiJsaWJyYXJpZXNfe21hbmFnZXJ9Il0gPSAiICIuam9pbihsaWJyYXJpZXMuZ2V0KG1hbmFnZXIsIFtdKSkKCiAgICAjIHNldCAiYXV0byIgY29uZGEgdmFsdWUKICAgIGlmIGl0ZW1bImNvbmRhIl0gPT0gImF1dG8iOgogICAgICAgIGl0ZW1bImNvbmRhIl0gPSAidHJ1ZSIgaWYgImNvbmRhIiBpbiBpdGVtWyJ0b3hlbnYiXSBlbHNlICJmYWxzZSIKCiAgICAjIGluamVjdCB0b3hkZXBzIGZvciBjb25kYQogICAgaWYgaXRlbVsiY29uZGEiXSA9PSAidHJ1ZSIgYW5kICJ0b3gtY29uZGEiIG5vdCBpbiBpdGVtWyJ0b3hkZXBzIl0ubG93ZXIoKToKICAgICAgICBpdGVtWyJ0b3hkZXBzIl0gPSAoInRveC1jb25kYSAiICsgaXRlbVsidG94ZGVwcyJdKS5zdHJpcCgpCgogICAgIyB2ZXJpZnkgdmFsdWVzCiAgICBhc3NlcnQgaXRlbVsicHl0ZXN0Il0gaW4geyJ0cnVlIiwgImZhbHNlIn0KICAgIGFzc2VydCBpdGVtWyJjb25kYSJdIGluIHsidHJ1ZSIsICJmYWxzZSJ9CiAgICBhc3NlcnQgaXRlbVsiZGlzcGxheSJdIGluIHsidHJ1ZSIsICJmYWxzZSJ9CgogICAgcmV0dXJuIGl0ZW0KCgppZiBfX25hbWVfXyA9PSAiX19tYWluX18iOgogICAgbG9hZF90b3hfdGFyZ2V0cygpCg==
+      - run: cat tox_matrix.py
       - id: set-outputs
         run: |
-          python tools/tox_matrix.py --envs "${{ inputs.envs }}" --libraries "${{ inputs.libraries }}" \
+          python tox_matrix.py --envs "${{ inputs.envs }}" --libraries "${{ inputs.libraries }}" \
           --posargs "${{ inputs.posargs }}" --toxdeps "${{ inputs.toxdeps }}" \
           --toxargs "${{ inputs.toxargs }}" --pytest "${{ inputs.pytest }}" \
           --coverage "${{ inputs.coverage }}" --conda "${{ inputs.conda }}" \

.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+repos:
+  - repo: https://github.com/PyCQA/flake8
+    rev: 4.0.1
+    hooks:
+      - id: flake8
+  - repo: https://github.com/myint/autoflake
+    rev: v1.4
+    hooks:
+      - id: autoflake
+        args:
+          - "--in-place"
+          - "--remove-all-unused-imports"
+          - "--remove-unused-variable"
+  - repo: https://github.com/PyCQA/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.1.0
+    hooks:
+      - id: check-ast
+      - id: check-case-conflict
+      - id: trailing-whitespace
+      - id: check-yaml
+      - id: check-added-large-files
+      - id: end-of-file-fixer
+      - id: mixed-line-ending
+  - repo: local
+    hooks:
+      - id: encode-scripts
+        name: encode scripts in workflows
+        language: system
+        entry: python update_scripts_in_yml.py
+        always_run: true

setup.cfg

@@ -7,3 +7,6 @@ packages = find:
 [options.extras_require]
 test =
     pytest
+
+[flake8]
+max-line-length = 100

setup.py

@@ -1,4 +1,5 @@
 import os
+
 from setuptools import Extension, setup
 
 setup(ext_modules=[Extension('test_package.simple',

update_scripts_in_yml.py

@@ -0,0 +1,30 @@
+import os
+from base64 import b64encode
+
+
+def base64_encode_into(script, yml_file, env_var):
+
+    with open(os.path.join('tools', script), 'rb') as f:
+        tox_matrix_base64 = b64encode(f.read()).decode('ascii')
+
+    with open(os.path.join('.github', 'workflows', yml_file)) as f:
+        tox_yml = f.read()
+
+    tox_yml_lines = tox_yml.splitlines()
+
+    for i in range(len(tox_yml_lines)):
+        if tox_yml_lines[i].strip().startswith(env_var + ':'):
+            pos = tox_yml_lines[i].index(':')
+            tox_yml_lines[i] = tox_yml_lines[i][:pos+1] + ' ' + tox_matrix_base64
+            break
+    else:
+        raise ValueError(f'No line containing {env_var} found')
+
+    tox_yml_new = '\n'.join(tox_yml_lines) + '\n'
+
+    with open(os.path.join('.github', 'workflows', yml_file), 'w') as f:
+        f.write(tox_yml_new)
+
+
+base64_encode_into('tox_matrix.py', 'tox.yml', 'TOX_MATRIX_SCRIPT')
+base64_encode_into('load_build_targets.py', 'publish.yml', 'LOAD_BUILD_TARGETS_SCRIPT')