Merge pull request #20 from simahawk/add-branch

Add new tool: oca-repo-add-branch

README.md

@@ -15,6 +15,14 @@ Features:
 * create/update repositories
 * create/update teams and roles
 * create/update branches
+* add new branches to existing YAML conf
+
+
+## Available tools
+
+* ``oca-repo-manage`` used to automatically maintain repositories based on YAML conf (see OCA conf below)
+* ``oca-repo-pages`` used to automatically generate repo inventory docs from the same YAML conf
+* ``oca-repo-add-branch`` used to manually add new branches to existing conf
 
 ## I can use it on my own organization?
 
@@ -29,6 +37,38 @@ https://github.com/OCA/repo-maintainer-conf
 
 You can use the script `scripts/bootstrap_data.py` to generate the conf out of existing repos. Run it with `--help` to see the options.
 
+# Usage
+
+## Manage repos
+
+This action is normally performed via GH actions in the conf repo. You should not run it manually.
+
+Yet, here's the command:
+
+
+    oca-repo-manage --org $GITHUB_REPOSITORY_OWNER --token ${{secrets.GIT_PUSH_TOKEN}} --conf-dir ./conf
+
+## Generate docs
+
+This action is normally performed via GH actions in the conf repo. You should not run it manually.
+
+Yet, here's the command:
+
+    oca-repo-pages --org $GITHUB_REPOSITORY_OWNER --conf-dir conf --path docsource
+
+## Add new branches to all repos
+
+This action has to be performed manually when you need a new branch to be added to all repos in your conf.
+Eg: when a new Odoo version is released.
+
+Go to the conf repo on your file system and run this:
+
+    oca-repo-add-branch --conf-dir ./conf/ --branch 18.0
+
+Review, stage all the changes, commit and open a PR.
+
+You can prevent this tool to edit a repo by adding ``manual_branch_mgmt`` boolean flag to repo's conf.
+
 ## Licenses
 
 This repository is licensed under [AGPL-3.0](LICENSE).

oca_repo_maintainer/cli/manage.py

@@ -6,6 +6,7 @@
 
 import click
 
+from ..tools.conf_file_manager import ConfFileManager
 from ..tools.manager import RepoManager
 
 
@@ -15,15 +16,24 @@
     "--token", required=True, prompt="Your github token", envvar="GITHUB_TOKEN"
 )
 @click.option(
-    # FIXME: switch to OCA when ready
     "--org",
     default="OCA",
     prompt="Your organization",
     help="The organizattion.",
 )
 def manage(conf_dir, org, token):
+    """Setup and update repositories and teams."""
     RepoManager(conf_dir, org, token).run()
 
 
+@click.command()
+@click.option("--conf-dir", required=True, help="Folder where configuration is stored")
+@click.option("--branch", required=True, help="New branch name to add")
+@click.option("--default", default=True, help="Set default branch as default.")
+def add_branch(conf_dir, branch, default=True):
+    """Add a branch to all repositories in the configuration."""
+    ConfFileManager(conf_dir).add_branch(branch, default=default)
+
+
 if __name__ == "__main__":
     manage()

oca_repo_maintainer/cli/pages.py

@@ -13,7 +13,6 @@
 @click.option("--conf-dir", required=True, help="Folder where configuration is stored")
 @click.option("--path", required=True, help="Folder where pages must be generated")
 @click.option(
-    # FIXME: switch to OCA when ready
     "--org",
     default="OCA",
     prompt="Your organization",

oca_repo_maintainer/tools/conf_file_manager.py

@@ -0,0 +1,66 @@
+# Copyright 2024 Camptocamp SA
+# @author: Simone Orsi
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl)
+
+import logging
+import sys
+
+from .utils import ConfLoader
+
+handler = logging.StreamHandler(sys.stdout)
+handler.setLevel(logging.INFO)
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
+handler.setFormatter(formatter)
+logging.basicConfig(level=logging.INFO, handlers=[handler])
+
+_logger = logging.getLogger(__name__)
+
+
+class ConfFileManager:
+    """Update existing configuration files."""
+
+    def __init__(self, conf_dir):
+        self.conf_dir = conf_dir
+        self.conf_loader = ConfLoader(self.conf_dir)
+        self.conf_repo = self.conf_loader.load_conf(
+            "repo", checksum=False, by_filepath=True
+        )
+
+    def add_branch(self, branch, default=True):
+        """Add a branch to all repositories in the configuration."""
+        for filepath, repo in self.conf_repo.items():
+            for repo_data in repo.values():
+                if self._has_manual_branch_mgmt(repo_data):
+                    _logger.info(
+                        "Skipping repo %s as manual_branch_mgmt is enabled.",
+                        filepath.as_posix(),
+                    )
+                    continue
+                if self._can_add_new_branch(branch, repo_data):
+                    repo_data["branches"].append(branch)
+                if default and self._can_change_default_branch(repo_data):
+                    repo_data["default_branch"] = branch
+            self.conf_loader.save_conf(filepath, repo)
+            _logger.info("Branch %s added to %s.", branch, filepath.as_posix())
+
+    def _has_manual_branch_mgmt(self, repo_data):
+        return repo_data.get("manual_branch_mgmt")
+
+    frozen_branches = ("master", "main")
+
+    def _can_add_new_branch(self, branch, repo_data):
+        branches = repo_data["branches"]
+        return (
+            branch not in branches
+            and all(x not in branches for x in self.frozen_branches)
+            and repo_data.get("default_branch") not in self.frozen_branches
+        )
+
+    def _can_change_default_branch(self, repo_data):
+        return (
+            # Only change if default branch is controlled via config file
+            "default_branch" in repo_data
+            # If the branch is "master" it means this is likely the repo of a tool
+            # and we have only one working branch.
+            and repo_data["default_branch"] not in self.frozen_branches
+        )

oca_repo_maintainer/tools/utils.py

@@ -29,19 +29,37 @@ def __init__(self, conf_dir):
     def _load_checksum(self):
         return self.load_conf("checksum", checksum=False)
 
-    def load_conf(self, name, checksum=True):
+    def load_conf(self, name, checksum=True, by_filepath=False):
         conf = {}
         path = self.conf_dir / name
         filepath = path.with_suffix(".yml")
         if filepath.exists():
             # direct yml files
-            conf.update(self._load_conf_from_file(filepath, checksum=checksum))
+            data = self._load_conf_from_file(filepath, checksum=checksum)
+            if by_filepath:
+                conf[filepath] = data
+            else:
+                conf.update(data)
         else:
             # folders containing ymls
             for filepath in path.rglob("*.yml"):
-                conf.update(self._load_conf_from_file(filepath, checksum=checksum))
+                data = self._load_conf_from_file(filepath, checksum=checksum)
+                if by_filepath:
+                    conf[filepath] = data
+                else:
+                    conf.update(data)
         return SmartDict(conf)
 
+    def save_conf(self, filepath, conf):
+        # TODO Use ruamel.yaml to keep the original format of the file.
+        # Yet, is not critical for now, because the pre-commit conf
+        # on repo-maintainer-conf will take care of it.
+        # However, it would be nice to have it here too.
+        with filepath.open("w") as f:
+            # at least keep the quotes consistent, you silly pyyaml
+            txt = yaml.dump(conf).replace(*"'", *'"')
+            f.write(txt)
+
     def _load_conf_from_file(self, filepath, checksum=True):
         conf = {}
         with filepath.open() as fd:

scripts/bootstrap_data.py

@@ -114,7 +114,6 @@ def prepare_repo(gh_org, conf_dir, whitelist=None):
 
 @click.command()
 @click.option(
-    # FIXME: switch to OCA when ready
     "--org",
     default="OCA",
     prompt="Your organization",

setup.py

@@ -61,6 +61,7 @@
     entry_points={
         "console_scripts": [
             "oca-repo-manage = oca_repo_maintainer.cli.manage:manage",
+            "oca-repo-add-branch = oca_repo_maintainer.cli.manage:add_branch",
             "oca-repo-pages = oca_repo_maintainer.cli.pages:pages",
         ]
     },

tests/__init__.py

@@ -1 +1,2 @@
 from . import test_manager
+from . import test_conf_file_manager

tests/common.py

@@ -10,6 +10,7 @@
 
 conf_path = Path(__file__).parent / "conf"
 conf_path2 = Path(__file__).parent / "conf2"
+conf_path_with_tools = Path(__file__).parent / "conf_with_tools"
 cassettes_path = Path(__file__).parent / "cassettes"
 
 

tests/conf_with_tools/repo/repo_for_addons.yml

@@ -0,0 +1,23 @@
+# For testing: this repo must exist already
+# but have a different default branch
+test-repo-for-addons:
+  name: Test repo for addons
+  description: Repo used to run real tests on oca-repo-manage tool.
+  category: Logistics
+  psc: test-team-1
+  maintainers: []
+  default_branch: "16.0"
+  branches:
+    - "16.0"
+    - "15.0"
+test-repo-for-addons-manual:
+  name: Test repo for addons with manual mgmt flag
+  description: Repo used to run real tests on oca-repo-manage tool.
+  category: Logistics
+  psc: test-team-1
+  maintainers: []
+  default_branch: "16.0"
+  branches:
+    - "16.0"
+    - "15.0"
+  manual_branch_mgmt: true

tests/conf_with_tools/repo/repo_for_tools.yml

@@ -0,0 +1,29 @@
+# For testing: this repo must NOT exist to test creation
+test-repo-for-tools-1:
+  name: Repository for tools with branch = master
+  description: Repo used to run real tests on oca-repo-manage tool.
+  category: Accounting
+  psc: test-team-2
+  maintainers:
+    - simahawk
+  default_branch: "master"
+  branches:
+    - "master"
+test-repo-for-tools-2:
+  name: Repository for tools with branch = main
+  description: Repo used to run real tests on oca-repo-manage tool.
+  category: Accounting
+  psc: test-team-2
+  maintainers:
+    - simahawk
+  default_branch: "main"
+  branches: []
+test-repo-for-tools-with-no-branches:
+  name: Repository for tools
+  description: Repo used to run real tests on oca-repo-manage tool.
+  category: Accounting
+  psc: test-team-2
+  maintainers:
+    - simahawk
+  default_branch: "master"
+  branches: []

tests/test_conf_file_manager.py

@@ -0,0 +1,114 @@
+# Copyright 2024 Camptocamp SA
+# @author: Simone Orsi
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl)
+
+import shutil
+import tempfile
+from unittest import TestCase
+
+from oca_repo_maintainer.tools.conf_file_manager import ConfFileManager
+
+from .common import conf_path, conf_path_with_tools
+
+
+class TestManager(TestCase):
+    def test_add_branch(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            shutil.copytree(conf_path.as_posix(), temp_dir, dirs_exist_ok=True)
+            manager = ConfFileManager(temp_dir)
+            manager.add_branch("100.0")
+
+            conf = manager.conf_loader.load_conf("repo")
+            self.assertTrue(conf)
+            for __, repo_data in conf.items():
+                self.assertIn("100.0", repo_data["branches"])
+                if "default_branch" in repo_data:
+                    self.assertEqual(repo_data["default_branch"], "100.0")
+
+    def test_add_branch_no_default(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            shutil.copytree(conf_path.as_posix(), temp_dir, dirs_exist_ok=True)
+            manager = ConfFileManager(temp_dir)
+            manager.add_branch("100.0", default=False)
+
+            conf = manager.conf_loader.load_conf("repo")
+            self.assertTrue(conf)
+            for __, repo_data in conf.items():
+                self.assertIn("100.0", repo_data["branches"])
+                if "default_branch" in repo_data:
+                    self.assertNotEqual(repo_data["default_branch"], "100.0")
+
+    def test_preserve_master(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            shutil.copytree(
+                conf_path_with_tools.as_posix(), temp_dir, dirs_exist_ok=True
+            )
+            manager = ConfFileManager(temp_dir)
+            conf = manager.conf_loader.load_conf("repo")
+
+            self.assertEqual(conf["test-repo-for-addons"]["branches"], ["16.0", "15.0"])
+            self.assertEqual(conf["test-repo-for-addons"]["default_branch"], "16.0")
+            self.assertEqual(conf["test-repo-for-tools-1"]["branches"], ["master"])
+            self.assertEqual(conf["test-repo-for-tools-1"]["default_branch"], "master")
+            self.assertEqual(conf["test-repo-for-tools-2"]["branches"], [])
+            self.assertEqual(conf["test-repo-for-tools-2"]["default_branch"], "main")
+            self.assertEqual(
+                conf["test-repo-for-tools-with-no-branches"]["branches"], []
+            )
+            self.assertEqual(
+                conf["test-repo-for-tools-with-no-branches"]["default_branch"], "master"
+            )
+
+            manager.add_branch("100.0")
+
+            conf = manager.conf_loader.load_conf("repo")
+
+            self.assertEqual(
+                conf["test-repo-for-addons"]["branches"], ["16.0", "15.0", "100.0"]
+            )
+            self.assertEqual(conf["test-repo-for-addons"]["default_branch"], "100.0")
+            self.assertEqual(conf["test-repo-for-tools-1"]["branches"], ["master"])
+            self.assertEqual(conf["test-repo-for-tools-1"]["default_branch"], "master")
+            self.assertEqual(conf["test-repo-for-tools-2"]["branches"], [])
+            self.assertEqual(conf["test-repo-for-tools-2"]["default_branch"], "main")
+            self.assertEqual(
+                conf["test-repo-for-tools-with-no-branches"]["branches"], []
+            )
+            self.assertEqual(
+                conf["test-repo-for-tools-with-no-branches"]["default_branch"], "master"
+            )
+
+    def test_skip_manual_mgmt(self):
+        with tempfile.TemporaryDirectory() as temp_dir:
+            shutil.copytree(
+                conf_path_with_tools.as_posix(), temp_dir, dirs_exist_ok=True
+            )
+            manager = ConfFileManager(temp_dir)
+            conf = manager.conf_loader.load_conf("repo")
+
+            self.assertEqual(conf["test-repo-for-addons"]["branches"], ["16.0", "15.0"])
+            self.assertEqual(conf["test-repo-for-addons"]["default_branch"], "16.0")
+            self.assertEqual(
+                conf["test-repo-for-addons-manual"]["branches"], ["16.0", "15.0"]
+            )
+            self.assertEqual(
+                conf["test-repo-for-addons-manual"]["default_branch"], "16.0"
+            )
+            self.assertEqual(
+                conf["test-repo-for-addons-manual"]["manual_branch_mgmt"], True
+            )
+
+            manager.add_branch("100.0")
+
+            conf = manager.conf_loader.load_conf("repo")
+
+            self.assertEqual(
+                conf["test-repo-for-addons"]["branches"], ["16.0", "15.0", "100.0"]
+            )
+            self.assertEqual(conf["test-repo-for-addons"]["default_branch"], "100.0")
+            self.assertEqual(
+                conf["test-repo-for-addons-manual"]["branches"], ["16.0", "15.0"]
+            )
+            self.assertEqual(
+                conf["test-repo-for-addons-manual"]["default_branch"], "16.0"
+            )