diff --git a/LICENSE b/LICENSE index 4a22480..e69de29 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2022 Andrew Lalis - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/pom.xml b/pom.xml index c1866d2..05947e3 100644 --- a/pom.xml +++ b/pom.xml @@ -6,7 +6,7 @@ nl.andrewl distribugit - 1.0.0 + 1.1.0 17 @@ -34,5 +34,38 @@ 1.7.36 runtime + + + org.kohsuke + github-api + 1.303 + + + + info.picocli + picocli + 4.6.3 + + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + + + nl.andrewl.distribugit.cli.DistribuGitCommand + + + + jar-with-dependencies + + + + + + \ No newline at end of file diff --git a/src/main/java/nl/andrewl/distribugit/DistribuGit.java b/src/main/java/nl/andrewl/distribugit/DistribuGit.java index 875e00a..952bab7 100644 --- a/src/main/java/nl/andrewl/distribugit/DistribuGit.java +++ b/src/main/java/nl/andrewl/distribugit/DistribuGit.java @@ -9,10 +9,37 @@ import java.nio.file.Path; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ForkJoinPool; +/** + * A single DistribuGit instance is used to execute a single sequence of + * operations on a set of git repositories, as configured according to its + * various components. + *

+ * A DistribuGit object, when invoking {@link DistribuGit#doActions()} or + * {@link DistribuGit#doActionsAsync()}, will perform the following series + * of operations: + *

+ *
    + *
  1. Call {@link RepositorySelector#getURIs()} to collect the list of + * repositories that it will operate on.
    2. + *
  2. Download each repository to the working directory.
    3. + *
  3. Applies the configured {@link RepositoryAction} to all of the + * repositories.
    4. + *
  4. If provided, applies the configured finalization action to all of + * the repositories.
    5. + *
  5. If needed, all repositories are deleted.
    6. + *
+ *

+ * Note that repositories are not guaranteed to be processed in any + * particular order. + *

+ */ public class DistribuGit { private final RepositorySelector selector; private final RepositoryAction action; + private final RepositoryAction finalizationAction; private final GitCredentials credentials; private final StatusListener statusListener; private final Path workingDir; @@ -22,9 +49,27 @@ public class DistribuGit { private int stepsComplete; private int stepsTotal; + /** + * Constructs a DistribuGit instance. + * @param selector A selector that provides a list of repository URIs. + * @param action An action to do for each repository. + * @param finalizationAction A final action to do for each repository, + * after all normal actions are done. + * @param credentials The credentials to use to operate on repositories. + * @param statusListener A listener that can be used to get information + * about the progress of the operations, and any + * messages that are emitted. + * @param workingDir The directory in which to do all git operations. + * @param strictFail Whether to fail instantly if any error occurs. If set + * to false, the program will continue even if actions + * fail for some repositories. + * @param cleanup Whether to perform cleanup after everything is done. This + * will remove the working directory once we're done. + */ public DistribuGit( RepositorySelector selector, RepositoryAction action, + RepositoryAction finalizationAction, GitCredentials credentials, StatusListener statusListener, Path workingDir, @@ -33,6 +78,7 @@ public class DistribuGit { ) { this.selector = selector; this.action = action; + this.finalizationAction = finalizationAction; this.credentials = credentials; this.statusListener = statusListener; this.workingDir = workingDir; @@ -40,16 +86,35 @@ public class DistribuGit { this.cleanup = cleanup; } - public void doActions() throws IOException { + /** + * Performs the configured actions on the selected git repositories. + * @throws IOException If an error occurs that requires us to quit early. + * This is only thrown if {@link DistribuGit#strictFail} is true. + */ + public synchronized void doActions() throws IOException { stepsComplete = 0; + if (Files.exists(workingDir)) { + try (var s = Files.list(workingDir)) { + if (s.findAny().isPresent()) throw new IOException("Working directory is not empty!"); + } + } Utils.delete(workingDir); // Delete the directory if it already exists. Files.createDirectory(workingDir); statusListener.messageReceived("Prepared temporary directory for repositories."); + List repositoryURIs; try { - List repositoryURIs = selector.getURIs(); - stepsTotal = 2 * repositoryURIs.size(); - Map repoDirs = downloadRepositories(repositoryURIs); - applyActionToRepositories(repoDirs); + repositoryURIs = selector.getURIs(); + } catch (Exception e) { + throw new IOException("Could not fetch repository URIs.", e); + } + try { + stepsTotal = 3 * repositoryURIs.size(); + Map repos = downloadRepositories(repositoryURIs); + applyActionToRepositories(repos, action); + if (finalizationAction != null) { + applyActionToRepositories(repos, finalizationAction); + } + repos.values().forEach(Git::close); } catch (Exception e) { throw new IOException(e); } finally { @@ -60,53 +125,93 @@ public class DistribuGit { } } + /** + * Runs the configured git actions on all selected repositories in an + * asynchronous manner. + * @return A future that completes when all actions are complete, or if an + * error occurs and the operation quits early. + */ + public CompletableFuture doActionsAsync() { + final CompletableFuture cf = new CompletableFuture<>(); + ForkJoinPool.commonPool().submit(() -> { + try { + doActions(); + cf.complete(null); + } catch (IOException e) { + cf.completeExceptionally(e); + } + }); + return cf; + } + private void completeStep() { stepsComplete++; statusListener.progressUpdated(stepsComplete / (float) stepsTotal * 100f); } - private Map downloadRepositories(List uris) throws IOException { - Map repositoryDirs = new HashMap<>(); + /** + * Downloads a set of repositories to working directories. + * @param uris The repositories to download. + * @return A map which maps each repository URI to its {@link Git} instance. + * @throws IOException If {@link DistribuGit#strictFail} is set to true, + * this will be thrown if an error occurs. + */ + private Map downloadRepositories(List uris) throws IOException { + Map repositoryDirs = new HashMap<>(); int dirIdx = 1; for (String repositoryURI : uris) { Path repoDir = workingDir.resolve(Integer.toString(dirIdx++)); + statusListener.messageReceived("Cloning repository " + repositoryURI + " to " + repoDir); + CloneCommand clone = Git.cloneRepository(); try { - statusListener.messageReceived("Cloning repository " + repositoryURI + " to " + repoDir); - CloneCommand clone = Git.cloneRepository(); credentials.addCredentials(clone); - clone.setDirectory(repoDir.toFile()); - clone.setURI(repositoryURI); - try (var ignored = clone.call()) { - repositoryDirs.put(repositoryURI, repoDir); - } catch (Exception e) { - if (strictFail) { - throw new IOException(e); - } else { - repositoryDirs.put(repositoryURI, null); - e.printStackTrace(); - } - } } catch (Exception e) { if (strictFail) { throw new IOException(e); } + statusListener.messageReceived("Could not add credentials to repository: " + e.getMessage()); e.printStackTrace(); + // Skip the rest of the logic since this failed. Just go to the next repository. + completeStep(); + continue; + } + clone.setDirectory(repoDir.toFile()); + clone.setURI(repositoryURI); + try (var git = clone.call()) { + repositoryDirs.put(repositoryURI, git); + } catch (Exception e) { + if (strictFail) { + throw new IOException(e); + } else { + statusListener.messageReceived("Could not clone repository: " + e.getMessage()); + repositoryDirs.put(repositoryURI, null); + e.printStackTrace(); + } } completeStep(); } return repositoryDirs; } - private void applyActionToRepositories(Map repoDirs) throws IOException { - for (var entry : repoDirs.entrySet()) { + /** + * Applies an action to all git repositories in the given map. + * @param repositories A map which maps URIs to {@link Git} instances. + * @param action The action to apply to each repository. + * @throws IOException If {@link DistribuGit#strictFail} is set to true, + * this will be thrown if an error occurs. + */ + private void applyActionToRepositories(Map repositories, RepositoryAction action) throws IOException { + for (var entry : repositories.entrySet()) { if (entry.getValue() != null) { - try (Git git = Git.open(entry.getValue().toFile())) { + try { + Git git = entry.getValue(); statusListener.messageReceived("Applying action to repository " + entry.getKey()); action.doAction(git); } catch (Exception e) { if (strictFail) { throw new IOException(e); } + statusListener.messageReceived("Action could not be applied to repository: " + e.getMessage()); e.printStackTrace(); } } else { @@ -116,9 +221,14 @@ public class DistribuGit { } } + /** + * A builder class to help with constructing {@link DistribuGit} instances + * with a fluent method interface. + */ public static class Builder { private RepositorySelector selector; private RepositoryAction action; + private RepositoryAction finalizationAction; private GitCredentials credentials = cmd -> {}; private StatusListener statusListener = new StatusListener() { @Override @@ -145,6 +255,11 @@ public class DistribuGit { return this; } + public Builder finalizationAction(RepositoryAction finalizationAction) { + this.finalizationAction = finalizationAction; + return this; + } + public Builder credentials(GitCredentials credentials) { this.credentials = credentials; return this; @@ -171,7 +286,10 @@ public class DistribuGit { } public DistribuGit build() { - return new DistribuGit(selector, action, credentials, statusListener, workingDir, strictFail, cleanup); + if (selector == null || action == null) { + throw new IllegalStateException("Cannot build an instance of DistribuGit without a selector or action."); + } + return new DistribuGit(selector, action, finalizationAction, credentials, statusListener, workingDir, strictFail, cleanup); } } } diff --git a/src/main/java/nl/andrewl/distribugit/GitCredentials.java b/src/main/java/nl/andrewl/distribugit/GitCredentials.java index 4aa182b..3a2721d 100644 --- a/src/main/java/nl/andrewl/distribugit/GitCredentials.java +++ b/src/main/java/nl/andrewl/distribugit/GitCredentials.java @@ -14,7 +14,17 @@ import org.eclipse.jgit.util.FS; import java.nio.charset.StandardCharsets; import java.nio.file.Path; +/** + * Supplies credentials information to a git command, which is needed when + * interacting with private repositories, or when pushing changes to a + * repository. + */ public interface GitCredentials { + /** + * Adds credentials information to the given command. + * @param gitCommand The command to add credentials to. + * @throws Exception If an error occurs while adding credentials. + */ void addCredentials(TransportCommand gitCommand) throws Exception; static GitCredentials ofUsernamePassword(String username, String password) { diff --git a/src/main/java/nl/andrewl/distribugit/RepositoryAction.java b/src/main/java/nl/andrewl/distribugit/RepositoryAction.java index 43b4fe9..21fe36e 100644 --- a/src/main/java/nl/andrewl/distribugit/RepositoryAction.java +++ b/src/main/java/nl/andrewl/distribugit/RepositoryAction.java @@ -2,7 +2,15 @@ package nl.andrewl.distribugit; import org.eclipse.jgit.api.Git; +/** + * An action that can be applied to a git repository. + */ public interface RepositoryAction { + /** + * Performs the action on the given git repository. + * @param git A reference to the git repository. + * @throws Exception If an error occurs during the action. + */ void doAction(Git git) throws Exception; /** diff --git a/src/main/java/nl/andrewl/distribugit/RepositorySelector.java b/src/main/java/nl/andrewl/distribugit/RepositorySelector.java index 8a052f9..b98452e 100644 --- a/src/main/java/nl/andrewl/distribugit/RepositorySelector.java +++ b/src/main/java/nl/andrewl/distribugit/RepositorySelector.java @@ -9,6 +9,11 @@ import java.util.List; * Component which produces a list of repositories to operate on. */ public interface RepositorySelector { + /** + * Gets a list of repository URIs to operate on. + * @return A list of repository URIs. + * @throws Exception If an error occurs while fetching the URIs. + */ List getURIs() throws Exception; static RepositorySelector fromCollection(Collection uris) { diff --git a/src/main/java/nl/andrewl/distribugit/StatusListener.java b/src/main/java/nl/andrewl/distribugit/StatusListener.java index 3535185..53b65b3 100644 --- a/src/main/java/nl/andrewl/distribugit/StatusListener.java +++ b/src/main/java/nl/andrewl/distribugit/StatusListener.java @@ -1,6 +1,18 @@ package nl.andrewl.distribugit; +/** + * Listens for updates during {@link DistribuGit#doActions()}. + */ public interface StatusListener { + /** + * Called when the operation's progress is updated. + * @param percentage The percentage (0 - 100) complete. + */ void progressUpdated(float percentage); + + /** + * Called when the DistribuGit operation emits a message. + * @param message The message that was emitted. + */ void messageReceived(String message); } diff --git a/src/main/java/nl/andrewl/distribugit/cli/DistribuGitCommand.java b/src/main/java/nl/andrewl/distribugit/cli/DistribuGitCommand.java new file mode 100644 index 0000000..0d0d4c4 --- /dev/null +++ b/src/main/java/nl/andrewl/distribugit/cli/DistribuGitCommand.java @@ -0,0 +1,85 @@ +package nl.andrewl.distribugit.cli; + +import nl.andrewl.distribugit.*; +import nl.andrewl.distribugit.selectors.GitHubSelectorBuilder; +import picocli.CommandLine; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.concurrent.Callable; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +@CommandLine.Command(name = "distribugit") +public class DistribuGitCommand implements Callable { + private static final Pattern SELECTOR_EXPRESSION_PATTERN = Pattern.compile("([\\w-]+):(.+)"); + private static final Pattern ORG_REPO_PREFIX_PATTERN = Pattern.compile("(.+)/(.+)"); + + @CommandLine.Option(names = {"-d", "--dir"}, description = "The working directory for DistribuGit", defaultValue = "./.distribugit_tmp") + private Path workingDir; + + @CommandLine.Option(names = {"-s", "--selector"}, description = "The repository selector to use. Format: \"slug:content\"", required = true) + private String selectorExpression; + + @CommandLine.Option(names = {"-a", "--action"}, description = "The command to run on each repository.", required = true) + private String actionCommand; + + @CommandLine.Option(names = {"-fa", "--finalization-action"}, description = "A command to run on each repository after all normal actions.") + private String finalizationActionCommand; + + @CommandLine.Option(names = {"-t", "--access-token"}, description = "The access token to use to perform operations.") + private String accessToken; + + @CommandLine.Option(names = {"-sf", "--strict-fail"}, description = "Whether to preemptively fail if any error occurs.", defaultValue = "true") + private boolean strictFail; + + @CommandLine.Option(names = {"-cl", "--cleanup"}, description = "Whether to remove all repository files when done.", defaultValue = "false") + private boolean cleanup; + + @Override + public Integer call() throws Exception { + var builder = new DistribuGit.Builder() + .workingDir(workingDir) + .strictFail(strictFail) + .cleanup(cleanup) + .selector(parseSelectorExpression(selectorExpression)) + .action(RepositoryAction.ofCommand(actionCommand.split("\\s+"))); + if (finalizationActionCommand != null) { + builder.finalizationAction(RepositoryAction.ofCommand(finalizationActionCommand.split("\\s+"))); + } + if (accessToken != null) { + builder.credentials(GitCredentials.ofUsernamePassword(accessToken, "")); + } + builder.statusListener(new StatusListener() { + @Override + public void progressUpdated(float percentage) { + System.out.printf("Progress: %.1f%%%n", percentage); + } + + @Override + public void messageReceived(String message) { + System.out.println(message); + } + }); + builder.build().doActions(); + return 0; + } + + private RepositorySelector parseSelectorExpression(String expr) throws IOException { + Matcher m = SELECTOR_EXPRESSION_PATTERN.matcher(expr); + if (!m.find()) throw new IllegalArgumentException("Invalid selector expression. Should be \"selector-type:expression\"."); + String slug = m.group(1); + String content = m.group(2); + if (slug.equalsIgnoreCase("org-repo-prefix")) { + Matcher m1 = ORG_REPO_PREFIX_PATTERN.matcher(content); + if (!m1.find()) throw new IllegalArgumentException("Invalid content for org-repo-prefix select. Should be \"orgName/prefix\""); + return GitHubSelectorBuilder.fromPersonalAccessToken(accessToken).orgAndPrefix(m1.group(1), m1.group(2)); + } else { + throw new IllegalArgumentException("Unsupported selector type: \"" + slug + "\"."); + } + } + + public static void main(String[] args) { + new CommandLine(new DistribuGitCommand()).execute(args); + } +} diff --git a/src/main/java/nl/andrewl/distribugit/selectors/GitHubRepositorySelector.java b/src/main/java/nl/andrewl/distribugit/selectors/GitHubRepositorySelector.java new file mode 100644 index 0000000..46d88ca --- /dev/null +++ b/src/main/java/nl/andrewl/distribugit/selectors/GitHubRepositorySelector.java @@ -0,0 +1,10 @@ +package nl.andrewl.distribugit.selectors; + +import org.kohsuke.github.GHRepository; +import org.kohsuke.github.GitHub; + +import java.util.List; + +public interface GitHubRepositorySelector { + List getRepos(GitHub gh) throws Exception; +} diff --git a/src/main/java/nl/andrewl/distribugit/selectors/GitHubSelectorBuilder.java b/src/main/java/nl/andrewl/distribugit/selectors/GitHubSelectorBuilder.java new file mode 100644 index 0000000..d02bef2 --- /dev/null +++ b/src/main/java/nl/andrewl/distribugit/selectors/GitHubSelectorBuilder.java @@ -0,0 +1,55 @@ +package nl.andrewl.distribugit.selectors; + +import nl.andrewl.distribugit.RepositorySelector; +import org.kohsuke.github.GHOrganization; +import org.kohsuke.github.GHRepository; +import org.kohsuke.github.GitHub; +import org.kohsuke.github.GitHubBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/** + * A builder that can be used to construct {@link RepositorySelector} instances + * that fetch repositories from GitHub via its API. + */ +public record GitHubSelectorBuilder(GitHub gh) { + public static GitHubSelectorBuilder fromPersonalAccessToken(String token) throws IOException { + return fromPersonalAccessToken(token, null); + } + + public static GitHubSelectorBuilder fromPersonalAccessToken(String token, String userOrOrgName) throws IOException { + return new GitHubSelectorBuilder(new GitHubBuilder().withOAuthToken(token, userOrOrgName).build()); + } + + /** + * Select repositories using an organization name, and a prefix to match + * against all repositories in that organization. + * @param orgName The name of the organization. + * @param prefix The prefix to use. + * @return A selector that selects matching repositories. + */ + public RepositorySelector orgAndPrefix(String orgName, String prefix) { + return () -> { + List repoURIs = new ArrayList<>(); + GHOrganization org = gh.getOrganization(orgName); + for (GHRepository repo : org.listRepositories()) { + if (repo.getName().startsWith(prefix)) { + repoURIs.add(repo.getHttpTransportUrl()); + } + } + return repoURIs; + }; + } + + /** + * A custom selector that can be used to perform some operations using the + * GitHub API and return a list of repositories. + * @param selector The GitHub selector to use. + * @return A selector that selects GitHub repositories. + */ + public RepositorySelector custom(GitHubRepositorySelector selector) { + return () -> selector.getRepos(gh).stream().map(GHRepository::getHttpTransportUrl).toList(); + } +}