Added CLI tool, and more GitHub stuff.

LICENSE

@@ -1,21 +0,0 @@
-MIT License
-
-Copyright (c) 2022 Andrew Lalis
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>nl.andrewl</groupId>
     <artifactId>distribugit</artifactId>
-    <version>1.0.0</version>
+    <version>1.1.0</version>
 
     <properties>
         <maven.compiler.source>17</maven.compiler.source>
@@ -34,5 +34,38 @@
             <version>1.7.36</version>
             <scope>runtime</scope>
         </dependency>
+        <!-- https://mvnrepository.com/artifact/org.kohsuke/github-api -->
+        <dependency>
+            <groupId>org.kohsuke</groupId>
+            <artifactId>github-api</artifactId>
+            <version>1.303</version>
+        </dependency>
+        <!-- https://mvnrepository.com/artifact/info.picocli/picocli -->
+        <dependency>
+            <groupId>info.picocli</groupId>
+            <artifactId>picocli</artifactId>
+            <version>4.6.3</version>
+        </dependency>
     </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.3.0</version>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <mainClass>nl.andrewl.distribugit.cli.DistribuGitCommand</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+
 </project>

src/main/java/nl/andrewl/distribugit/DistribuGit.java

@@ -9,10 +9,37 @@ import java.nio.file.Path;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ForkJoinPool;
 
+/**
+ * A single DistribuGit instance is used to execute a single sequence of
+ * operations on a set of git repositories, as configured according to its
+ * various components.
+ * <p>
+ *     A DistribuGit object, when invoking {@link DistribuGit#doActions()} or
+ *     {@link DistribuGit#doActionsAsync()}, will perform the following series
+ *     of operations:
+ * </p>
+ * <ol>
+ *     <li>Call {@link RepositorySelector#getURIs()} to collect the list of
+ *     repositories that it will operate on.</li>
+ *     <li>Download each repository to the working directory.</li>
+ *     <li>Applies the configured {@link RepositoryAction} to all of the
+ *     repositories.</li>
+ *     <li>If provided, applies the configured finalization action to all of
+ *     the repositories.</li>
+ *     <li>If needed, all repositories are deleted.</li>
+ * </ol>
+ * <p>
+ *     Note that repositories are not guaranteed to be processed in any
+ *     particular order.
+ * </p>
+ */
 public class DistribuGit {
 	private final RepositorySelector selector;
 	private final RepositoryAction action;
+	private final RepositoryAction finalizationAction;
 	private final GitCredentials credentials;
 	private final StatusListener statusListener;
 	private final Path workingDir;
@@ -22,9 +49,27 @@ public class DistribuGit {
 	private int stepsComplete;
 	private int stepsTotal;
 
+	/**
+	 * Constructs a DistribuGit instance.
+	 * @param selector A selector that provides a list of repository URIs.
+	 * @param action An action to do for each repository.
+	 * @param finalizationAction A final action to do for each repository,
+	 *                           after all normal actions are done.
+	 * @param credentials The credentials to use to operate on repositories.
+	 * @param statusListener A listener that can be used to get information
+	 *                       about the progress of the operations, and any
+	 *                       messages that are emitted.
+	 * @param workingDir The directory in which to do all git operations.
+	 * @param strictFail Whether to fail instantly if any error occurs. If set
+	 *                   to false, the program will continue even if actions
+	 *                   fail for some repositories.
+	 * @param cleanup Whether to perform cleanup after everything is done. This
+	 *                will remove the working directory once we're done.
+	 */
 	public DistribuGit(
 		RepositorySelector selector,
 		RepositoryAction action,
+		RepositoryAction finalizationAction,
 		GitCredentials credentials,
 		StatusListener statusListener,
 		Path workingDir,
@@ -33,6 +78,7 @@ public class DistribuGit {
 	) {
 		this.selector = selector;
 		this.action = action;
+		this.finalizationAction = finalizationAction;
 		this.credentials = credentials;
 		this.statusListener = statusListener;
 		this.workingDir = workingDir;
@@ -40,16 +86,35 @@ public class DistribuGit {
 		this.cleanup = cleanup;
 	}
 
-	public void doActions() throws IOException {
+	/**
+	 * Performs the configured actions on the selected git repositories.
+	 * @throws IOException If an error occurs that requires us to quit early.
+	 * This is only thrown if {@link DistribuGit#strictFail} is true.
+	 */
+	public synchronized void doActions() throws IOException {
 		stepsComplete = 0;
+		if (Files.exists(workingDir)) {
+			try (var s = Files.list(workingDir)) {
+				if (s.findAny().isPresent()) throw new IOException("Working directory is not empty!");
+			}
+		}
 		Utils.delete(workingDir); // Delete the directory if it already exists.
 		Files.createDirectory(workingDir);
 		statusListener.messageReceived("Prepared temporary directory for repositories.");
+		List<String> repositoryURIs;
 		try {
-			List<String> repositoryURIs = selector.getURIs();
+			repositoryURIs = selector.getURIs();
-			stepsTotal = 2 * repositoryURIs.size();
+		} catch (Exception e) {
-			Map<String, Path> repoDirs = downloadRepositories(repositoryURIs);
+			throw new IOException("Could not fetch repository URIs.", e);
-			applyActionToRepositories(repoDirs);
+		}
+		try {
+			stepsTotal = 3 * repositoryURIs.size();
+			Map<String, Git> repos = downloadRepositories(repositoryURIs);
+			applyActionToRepositories(repos, action);
+			if (finalizationAction != null) {
+				applyActionToRepositories(repos, finalizationAction);
+			}
+			repos.values().forEach(Git::close);
 		} catch (Exception e) {
 			throw new IOException(e);
 		} finally {
@@ -60,53 +125,93 @@ public class DistribuGit {
 		}
 	}
 
+	/**
+	 * Runs the configured git actions on all selected repositories in an
+	 * asynchronous manner.
+	 * @return A future that completes when all actions are complete, or if an
+	 * error occurs and the operation quits early.
+	 */
+	public CompletableFuture<Void> doActionsAsync() {
+		final CompletableFuture<Void> cf = new CompletableFuture<>();
+		ForkJoinPool.commonPool().submit(() -> {
+			try {
+				doActions();
+				cf.complete(null);
+			} catch (IOException e) {
+				cf.completeExceptionally(e);
+			}
+		});
+		return cf;
+	}
+
 	private void completeStep() {
 		stepsComplete++;
 		statusListener.progressUpdated(stepsComplete / (float) stepsTotal * 100f);
 	}
 
-	private Map<String, Path> downloadRepositories(List<String> uris) throws IOException {
+	/**
-		Map<String, Path> repositoryDirs = new HashMap<>();
+	 * Downloads a set of repositories to working directories.
+	 * @param uris The repositories to download.
+	 * @return A map which maps each repository URI to its {@link Git} instance.
+	 * @throws IOException If {@link DistribuGit#strictFail} is set to true,
+	 * this will be thrown if an error occurs.
+	 */
+	private Map<String, Git> downloadRepositories(List<String> uris) throws IOException {
+		Map<String, Git> repositoryDirs = new HashMap<>();
 		int dirIdx = 1;
 		for (String repositoryURI : uris) {
 			Path repoDir = workingDir.resolve(Integer.toString(dirIdx++));
-			try {
 			statusListener.messageReceived("Cloning repository " + repositoryURI + " to " + repoDir);
 			CloneCommand clone = Git.cloneRepository();
+			try {
 				credentials.addCredentials(clone);
+			} catch (Exception e) {
+				if (strictFail) {
+					throw new IOException(e);
+				}
+				statusListener.messageReceived("Could not add credentials to repository: " + e.getMessage());
+				e.printStackTrace();
+				// Skip the rest of the logic since this failed. Just go to the next repository.
+				completeStep();
+				continue;
+			}
 			clone.setDirectory(repoDir.toFile());
 			clone.setURI(repositoryURI);
-				try (var ignored = clone.call()) {
+			try (var git = clone.call()) {
-					repositoryDirs.put(repositoryURI, repoDir);
+				repositoryDirs.put(repositoryURI, git);
 			} catch (Exception e) {
 				if (strictFail) {
 					throw new IOException(e);
 				} else {
+					statusListener.messageReceived("Could not clone repository: " + e.getMessage());
 					repositoryDirs.put(repositoryURI, null);
 					e.printStackTrace();
 				}
 			}
-			} catch (Exception e) {
-				if (strictFail) {
-					throw new IOException(e);
-				}
-				e.printStackTrace();
-			}
 			completeStep();
 		}
 		return repositoryDirs;
 	}
 
-	private void applyActionToRepositories(Map<String, Path> repoDirs) throws IOException {
+	/**
-		for (var entry : repoDirs.entrySet()) {
+	 * Applies an action to all git repositories in the given map.
+	 * @param repositories A map which maps URIs to {@link Git} instances.
+	 * @param action The action to apply to each repository.
+	 * @throws IOException If {@link DistribuGit#strictFail} is set to true,
+	 * this will be thrown if an error occurs.
+	 */
+	private void applyActionToRepositories(Map<String, Git> repositories, RepositoryAction action) throws IOException {
+		for (var entry : repositories.entrySet()) {
 			if (entry.getValue() != null) {
-				try (Git git = Git.open(entry.getValue().toFile())) {
+				try {
+					Git git = entry.getValue();
 					statusListener.messageReceived("Applying action to repository " + entry.getKey());
 					action.doAction(git);
 				} catch (Exception e) {
 					if (strictFail) {
 						throw new IOException(e);
 					}
+					statusListener.messageReceived("Action could not be applied to repository: " + e.getMessage());
 					e.printStackTrace();
 				}
 			} else {
@@ -116,9 +221,14 @@ public class DistribuGit {
 		}
 	}
 
+	/**
+	 * A builder class to help with constructing {@link DistribuGit} instances
+	 * with a fluent method interface.
+	 */
 	public static class Builder {
 		private RepositorySelector selector;
 		private RepositoryAction action;
+		private RepositoryAction finalizationAction;
 		private GitCredentials credentials = cmd -> {};
 		private StatusListener statusListener = new StatusListener() {
 			@Override
@@ -145,6 +255,11 @@ public class DistribuGit {
 			return this;
 		}
 
+		public Builder finalizationAction(RepositoryAction finalizationAction) {
+			this.finalizationAction = finalizationAction;
+			return this;
+		}
+
 		public Builder credentials(GitCredentials credentials) {
 			this.credentials = credentials;
 			return this;
@@ -171,7 +286,10 @@ public class DistribuGit {
 		}
 
 		public DistribuGit build() {
-			return new DistribuGit(selector, action, credentials, statusListener, workingDir, strictFail, cleanup);
+			if (selector == null || action == null) {
+				throw new IllegalStateException("Cannot build an instance of DistribuGit without a selector or action.");
+			}
+			return new DistribuGit(selector, action, finalizationAction, credentials, statusListener, workingDir, strictFail, cleanup);
 		}
 	}
 }

src/main/java/nl/andrewl/distribugit/GitCredentials.java

@@ -14,7 +14,17 @@ import org.eclipse.jgit.util.FS;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Path;
 
+/**
+ * Supplies credentials information to a git command, which is needed when
+ * interacting with private repositories, or when pushing changes to a
+ * repository.
+ */
 public interface GitCredentials {
+	/**
+	 * Adds credentials information to the given command.
+	 * @param gitCommand The command to add credentials to.
+	 * @throws Exception If an error occurs while adding credentials.
+	 */
 	void addCredentials(TransportCommand<?, ?> gitCommand) throws Exception;
 
 	static GitCredentials ofUsernamePassword(String username, String password) {

src/main/java/nl/andrewl/distribugit/RepositoryAction.java

@@ -2,7 +2,15 @@ package nl.andrewl.distribugit;
 
 import org.eclipse.jgit.api.Git;
 
+/**
+ * An action that can be applied to a git repository.
+ */
 public interface RepositoryAction {
+	/**
+	 * Performs the action on the given git repository.
+	 * @param git A reference to the git repository.
+	 * @throws Exception If an error occurs during the action.
+	 */
 	void doAction(Git git) throws Exception;
 
 	/**

src/main/java/nl/andrewl/distribugit/RepositorySelector.java

@@ -9,6 +9,11 @@ import java.util.List;
  * Component which produces a list of repositories to operate on.
  */
 public interface RepositorySelector {
+	/**
+	 * Gets a list of repository URIs to operate on.
+	 * @return A list of repository URIs.
+	 * @throws Exception If an error occurs while fetching the URIs.
+	 */
 	List<String> getURIs() throws Exception;
 
 	static RepositorySelector fromCollection(Collection<String> uris) {

src/main/java/nl/andrewl/distribugit/StatusListener.java

@@ -1,6 +1,18 @@
 package nl.andrewl.distribugit;
 
+/**
+ * Listens for updates during {@link DistribuGit#doActions()}.
+ */
 public interface StatusListener {
+	/**
+	 * Called when the operation's progress is updated.
+	 * @param percentage The percentage (0 - 100) complete.
+	 */
 	void progressUpdated(float percentage);
+
+	/**
+	 * Called when the DistribuGit operation emits a message.
+	 * @param message The message that was emitted.
+	 */
 	void messageReceived(String message);
 }

src/main/java/nl/andrewl/distribugit/cli/DistribuGitCommand.java

@@ -0,0 +1,85 @@
+package nl.andrewl.distribugit.cli;
+
+import nl.andrewl.distribugit.*;
+import nl.andrewl.distribugit.selectors.GitHubSelectorBuilder;
+import picocli.CommandLine;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@CommandLine.Command(name = "distribugit")
+public class DistribuGitCommand implements Callable<Integer> {
+	private static final Pattern SELECTOR_EXPRESSION_PATTERN = Pattern.compile("([\\w-]+):(.+)");
+	private static final Pattern ORG_REPO_PREFIX_PATTERN = Pattern.compile("(.+)/(.+)");
+
+	@CommandLine.Option(names = {"-d", "--dir"}, description = "The working directory for DistribuGit", defaultValue = "./.distribugit_tmp")
+	private Path workingDir;
+
+	@CommandLine.Option(names = {"-s", "--selector"}, description = "The repository selector to use. Format: \"slug:content\"", required = true)
+	private String selectorExpression;
+
+	@CommandLine.Option(names = {"-a", "--action"}, description = "The command to run on each repository.", required = true)
+	private String actionCommand;
+
+	@CommandLine.Option(names = {"-fa", "--finalization-action"}, description = "A command to run on each repository after all normal actions.")
+	private String finalizationActionCommand;
+
+	@CommandLine.Option(names = {"-t", "--access-token"}, description = "The access token to use to perform operations.")
+	private String accessToken;
+
+	@CommandLine.Option(names = {"-sf", "--strict-fail"}, description = "Whether to preemptively fail if any error occurs.", defaultValue = "true")
+	private boolean strictFail;
+
+	@CommandLine.Option(names = {"-cl", "--cleanup"}, description = "Whether to remove all repository files when done.", defaultValue = "false")
+	private boolean cleanup;
+
+	@Override
+	public Integer call() throws Exception {
+		var builder = new DistribuGit.Builder()
+				.workingDir(workingDir)
+				.strictFail(strictFail)
+				.cleanup(cleanup)
+				.selector(parseSelectorExpression(selectorExpression))
+				.action(RepositoryAction.ofCommand(actionCommand.split("\\s+")));
+		if (finalizationActionCommand != null) {
+			builder.finalizationAction(RepositoryAction.ofCommand(finalizationActionCommand.split("\\s+")));
+		}
+		if (accessToken != null) {
+			builder.credentials(GitCredentials.ofUsernamePassword(accessToken, ""));
+		}
+		builder.statusListener(new StatusListener() {
+			@Override
+			public void progressUpdated(float percentage) {
+				System.out.printf("Progress: %.1f%%%n", percentage);
+			}
+
+			@Override
+			public void messageReceived(String message) {
+				System.out.println(message);
+			}
+		});
+		builder.build().doActions();
+		return 0;
+	}
+
+	private RepositorySelector parseSelectorExpression(String expr) throws IOException {
+		Matcher m = SELECTOR_EXPRESSION_PATTERN.matcher(expr);
+		if (!m.find()) throw new IllegalArgumentException("Invalid selector expression. Should be \"selector-type:expression\".");
+		String slug = m.group(1);
+		String content = m.group(2);
+		if (slug.equalsIgnoreCase("org-repo-prefix")) {
+			Matcher m1 = ORG_REPO_PREFIX_PATTERN.matcher(content);
+			if (!m1.find()) throw new IllegalArgumentException("Invalid content for org-repo-prefix select. Should be \"orgName/prefix\"");
+			return GitHubSelectorBuilder.fromPersonalAccessToken(accessToken).orgAndPrefix(m1.group(1), m1.group(2));
+		} else {
+			throw new IllegalArgumentException("Unsupported selector type: \"" + slug + "\".");
+		}
+	}
+
+	public static void main(String[] args) {
+		new CommandLine(new DistribuGitCommand()).execute(args);
+	}
+}

src/main/java/nl/andrewl/distribugit/selectors/GitHubRepositorySelector.java

@@ -0,0 +1,10 @@
+package nl.andrewl.distribugit.selectors;
+
+import org.kohsuke.github.GHRepository;
+import org.kohsuke.github.GitHub;
+
+import java.util.List;
+
+public interface GitHubRepositorySelector {
+	List<GHRepository> getRepos(GitHub gh) throws Exception;
+}

src/main/java/nl/andrewl/distribugit/selectors/GitHubSelectorBuilder.java

@@ -0,0 +1,55 @@
+package nl.andrewl.distribugit.selectors;
+
+import nl.andrewl.distribugit.RepositorySelector;
+import org.kohsuke.github.GHOrganization;
+import org.kohsuke.github.GHRepository;
+import org.kohsuke.github.GitHub;
+import org.kohsuke.github.GitHubBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * A builder that can be used to construct {@link RepositorySelector} instances
+ * that fetch repositories from GitHub via its API.
+ */
+public record GitHubSelectorBuilder(GitHub gh) {
+	public static GitHubSelectorBuilder fromPersonalAccessToken(String token) throws IOException {
+		return fromPersonalAccessToken(token, null);
+	}
+
+	public static GitHubSelectorBuilder fromPersonalAccessToken(String token, String userOrOrgName) throws IOException {
+		return new GitHubSelectorBuilder(new GitHubBuilder().withOAuthToken(token, userOrOrgName).build());
+	}
+
+	/**
+	 * Select repositories using an organization name, and a prefix to match
+	 * against all repositories in that organization.
+	 * @param orgName The name of the organization.
+	 * @param prefix The prefix to use.
+	 * @return A selector that selects matching repositories.
+	 */
+	public RepositorySelector orgAndPrefix(String orgName, String prefix) {
+		return () -> {
+			List<String> repoURIs = new ArrayList<>();
+			GHOrganization org = gh.getOrganization(orgName);
+			for (GHRepository repo : org.listRepositories()) {
+				if (repo.getName().startsWith(prefix)) {
+					repoURIs.add(repo.getHttpTransportUrl());
+				}
+			}
+			return repoURIs;
+		};
+	}
+
+	/**
+	 * A custom selector that can be used to perform some operations using the
+	 * GitHub API and return a list of repositories.
+	 * @param selector The GitHub selector to use.
+	 * @return A selector that selects GitHub repositories.
+	 */
+	public RepositorySelector custom(GitHubRepositorySelector selector) {
+		return () -> selector.getRepos(gh).stream().map(GHRepository::getHttpTransportUrl).toList();
+	}
+}