Add localregistry input option (#38)

* Add localregistry input option

Option to add local registries to build process by specifying their url (https). Authentication to the repos  via user GitHub token.

* Backward compatible keyword syntax and Wrap new code in VERSION > 1.5

* Update Interface&README, add git_cli amd remove token flag

Provide multiple remotes separated with |
Add example to ReadMe
Support for private repos/packages with SSH protocol

* Update README.md

Co-authored-by: Sascha Mann <git@mail.saschamann.eu>

* Update action.yml

Co-authored-by: Sascha Mann <git@mail.saschamann.eu>

* Print warn mesasge if git_cli = true with Julia<1.7.

* Add `::notice::` formatting

---------

Co-authored-by: Sascha Mann <git@mail.saschamann.eu>

.github/workflows/ci.yml

@@ -15,7 +15,6 @@ on:
 
 jobs:
   test:
-    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -31,6 +30,9 @@ jobs:
         arch:
           - x64
           - x86
+        pkg-server:
+          - ""
+          - "pkg.julialang.org"
         # 32-bit Julia binaries are not available on macOS
         exclude:
           - os: macOS-latest
@@ -68,6 +70,8 @@ jobs:
             ${{ runner.os }}-
 
       - uses: ./.github/actions/julia-buildpkg
+        env:
+          JULIA_PKG_SERVER: ${{ matrix.pkg-server }}
 
       - uses: julia-actions/julia-runtest@v1
 

README.md

@@ -33,3 +33,39 @@ jobs:
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
 ```
+
+
+### Registry flavor preference
+
+This actions defines (and exports for subsequent steps of the workflow) the
+environmental variable `JULIA_PKG_SERVER_REGISTRY_PREFERENCE=eager` unless it
+is already set. If you want another registry flavor (i.e. `conservative`) this
+should be defined in the `env:` section of the relevant workflow or step. See
+[Registry flavors](https://pkgdocs.julialang.org/dev/registries/#Registry-flavors)
+for more information.
+
+### Adding Local Registries
+
+Personal registries, e.g. created with [LocalRegistry.jl](https://github.com/GunnarFarneback/LocalRegistry.jl), can be added to the CI using the `localregistry` input option. If the personal registry as well as packages needed in the current project are public, no additional setup is required if the registry url is specified in https-format.
+
+If the registry contains private packages, or is itself private, the ssh protocol should to be used. The user has to provide the corresponding private SSH-keys to the `ssh-agent` to access packages and registry. This can be conveniently done using the [webfactory/ssh-agent](https://github.com/webfactory/ssh-agent) action. A snippet illustrating the usage of (private) personal registries is shown below
+
+```yaml
+...   
+      # Adding private SSH keys (only necessary for accessing private packages and/or 
+      # when providing Registry-link in ssh format)
+      - uses: webfactory/ssh-agent@v0.8.0
+        with:
+          ssh-private-key: |
+            ${{ secrets.PRIVATE_DEPLOY_KEY }}
+            ${{ secrets.PRIVATE_DEPLOY_KEY2 }}
+      - uses: julia-actions/julia-buildpkg@v1
+        with:
+          localregistry: |
+            https://github.com/username/PersonalRegistry.git
+            git@github.com:username2/PersonalRegistry2.git
+          git_cli: false # = JULIA_PKG_USE_CLI_GIT. Options: true | false (default)
+...
+```
+
+For Julia 1.7 and above, the `git_cli` option can be used to set the `JULIA_PKG_USE_CLI_GIT` [environment flag](https://docs.julialang.org/en/v1/manual/environment-variables/), for additional control of the SSH configuration used by `Pkg` to add/dev packages.

action.yml

@@ -13,28 +13,47 @@ inputs:
   precompile:
     description: 'Whether to allow auto-precompilation (via the `JULIA_PKG_PRECOMPILE_AUTO` env var). Options: yes | no. Default value: no.'
     default: 'no'
+  localregistry:
+    description: 'Add local registries hosted on GitHub. Specified by providing the url (https/ssh) to the repositories as a newline (\n) seperated list.
+                  User is responsible for setting up the necessary SSH-Keys to access the repositories if necessary.'
+    default: ''
+  git_cli:
+    description: 'Determine if Pkg uses the cli git executable (Julia >= 1.7). Might be necessary for more complicated SSH setups.
+                  Options: true | false. Default : false'
+    default: 'false'
 
 runs:
   using: 'composite'
   steps:
-    # Occasionally, there are rather large delays (> a few hours)
-    # between the time a package is registered in General and
-    # propagated to pkg.julialang.org.  We can avoid this by manually
-    # cloning ~/.julia/registries/General/ in Julia 1.5 and later.
-    # See:
-    # * https://github.com/JuliaLang/Pkg.jl/issues/2011
-    # * https://github.com/JuliaRegistries/General/issues/16777
-    # * https://github.com/JuliaPackaging/PkgServer.jl/issues/60
-  - run: julia --color=yes "$GITHUB_ACTION_PATH"/add_general_registry.jl
+  - name: Set and export registry flavor preference
+    run: echo "JULIA_PKG_SERVER_REGISTRY_PREFERENCE=${JULIA_PKG_SERVER_REGISTRY_PREFERENCE:-eager}" >> ${GITHUB_ENV}
     shell: bash
-    env:
-      # We set `JULIA_PKG_SERVER` only for this step to enforce
-      # `Pkg.Registry.add` to use Git.  This way, Pkg.jl can send
-      # the request metadata to pkg.julialang.org when installing
-      # packages via `Pkg.test`.
-      JULIA_PKG_SERVER: ""
+  - run: |
+      import Pkg
+      
+      # Determine if Pkg uses git-cli executable instead of LibGit2
+      VERSION >= v"1.7-" && (ENV["JULIA_PKG_USE_CLI_GIT"] = ${{ inputs.git_cli }})
 
-  - run: julia --color=yes --project=${{ inputs.project }} -e 'using Pkg; if VERSION >= v"1.1.0-rc1"; Pkg.build(verbose=true); else Pkg.build(); end'
-    shell: bash
+      if VERSION < v"1.7-" && ${{ inputs.git_cli }} == true
+        printstyled("::notice::JULIA_PKG_USE_CLI_GIT requires Julia >= 1.7. Using default LibGit2 git-interface instead! \n"; color = :yellow)
+      end
+
+
+      if VERSION >= v"1.5-"
+          Pkg.Registry.add("General")
+
+          # If provided add local registries
+          if !isempty("${{ inputs.localregistry }}")
+            local_repos = split("${{ inputs.localregistry }}", "\n") .|> string   
+            for repo_url in local_repos
+              isempty(repo_url) && continue
+              Pkg.Registry.add(Pkg.RegistrySpec(; url = repo_url))
+            end
+          end
+      end
+      
+      VERSION >= v"1.1.0-rc1" ? retry(Pkg.build)(verbose=true) : retry(Pkg.build)()
+    shell: julia --color=yes --project=${{ inputs.project }} {0}
     env:
       JULIA_PKG_PRECOMPILE_AUTO: "${{ inputs.precompile }}"
+      GITHUB_TOKEN: ${{ github.token }}

add_general_registry.jl

@@ -1,60 +0,0 @@
-using Pkg
-
-function tarball_general_registry_location()
-    reg_dir = joinpath(DEPOT_PATH[1], "registries")
-    general_registry_tarball = joinpath(reg_dir, "General.tar.gz")
-    registry_toml_file = joinpath(reg_dir, "General.toml")
-    return general_registry_tarball, registry_toml_file
-end
-
-function cloned_general_registry_location()
-    general_registry_dir = joinpath(DEPOT_PATH[1], "registries", "General")
-    registry_toml_file = joinpath(general_registry_dir, "Registry.toml")
-    return general_registry_dir, registry_toml_file
-end
-
-function general_registry_exists()
-    general_registry_tarball, registry_toml_file = tarball_general_registry_location()
-    if isfile(general_registry_tarball) && isfile(registry_toml_file)
-        return true
-    end
-    general_registry_dir, registry_toml_file = cloned_general_registry_location()
-    if !isdir(general_registry_dir)
-        return false
-    elseif !isfile(registry_toml_file)
-        return false
-    else
-        return true
-    end
-end
-
-function add_general_registry()
-    @info("Attempting to clone the General registry")
-    general_registry_dir, registry_toml_file = general_registry_location()
-    rm(general_registry_dir; force = true, recursive = true)
-    Pkg.Registry.add("General")
-    isfile(registry_toml_file) || throw(ErrorException("the Registry.toml file does not exist"))
-    return nothing
-end
-
-function main(; n = 10, max_delay = 120)
-    VERSION >= v"1.5-" || return
-
-    if general_registry_exists()
-        @info("The General registry already exists locally")
-        return
-    end
-
-    delays = ExponentialBackOff(; n = n, max_delay = max_delay)
-    try
-        retry(add_general_registry; delays = delays)()
-        @info("Successfully added the General registry")
-    catch ex
-        msg = "I was unable to add the General registry. However, the build will continue."
-        @error(msg, exception=(ex,catch_backtrace()))
-    end
-
-    return
-end
-
-main()