andrew
/
distribugit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added CLI tool, and more GitHub stuff.

This commit is contained in:
Andrew Lalis 2022-04-20 10:59:48 +02:00
parent 4f0848c2d1
commit 4df5d82c08
10 changed files with 362 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
LICENSE
View File

@ -1,21 +0,0 @@
MIT License
Copyright (c) 2022 Andrew Lalis
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

35
pom.xml
View File

@ -6,7 +6,7 @@
<groupId>nl.andrewl</groupId>
<artifactId>distribugit</artifactId>
<version>1.0.0</version>
<version>1.1.0</version>
<properties>
<maven.compiler.source>17</maven.compiler.source>
@ -34,5 +34,38 @@
<version>1.7.36</version>
<scope>runtime</scope>
</dependency>
<!-- https://mvnrepository.com/artifact/org.kohsuke/github-api -->
<dependency>
<groupId>org.kohsuke</groupId>
<artifactId>github-api</artifactId>
<version>1.303</version>
</dependency>
<!-- https://mvnrepository.com/artifact/info.picocli/picocli -->
<dependency>
<groupId>info.picocli</groupId>
<artifactId>picocli</artifactId>
<version>4.6.3</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>3.3.0</version>
<configuration>
<archive>
<manifest>
<mainClass>nl.andrewl.distribugit.cli.DistribuGitCommand</mainClass>
</manifest>
</archive>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
</configuration>
</plugin>
</plugins>
</build>
</project>

168
src/main/java/nl/andrewl/distribugit/DistribuGit.java
View File

@ -9,10 +9,37 @@ import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ForkJoinPool;
/**
* A single DistribuGit instance is used to execute a single sequence of
* operations on a set of git repositories, as configured according to its
* various components.
* <p>
* A DistribuGit object, when invoking {@link DistribuGit#doActions()} or
* {@link DistribuGit#doActionsAsync()}, will perform the following series
* of operations:
* </p>
* <ol>
* <li>Call {@link RepositorySelector#getURIs()} to collect the list of
* repositories that it will operate on.</li>
* <li>Download each repository to the working directory.</li>
* <li>Applies the configured {@link RepositoryAction} to all of the
* repositories.</li>
* <li>If provided, applies the configured finalization action to all of
* the repositories.</li>
* <li>If needed, all repositories are deleted.</li>
* </ol>
* <p>
* Note that repositories are not guaranteed to be processed in any
* particular order.
* </p>
*/
public class DistribuGit {
private final RepositorySelector selector;
private final RepositoryAction action;
private final RepositoryAction finalizationAction;
private final GitCredentials credentials;
private final StatusListener statusListener;
private final Path workingDir;
@ -22,9 +49,27 @@ public class DistribuGit {
private int stepsComplete;
private int stepsTotal;
/**
* Constructs a DistribuGit instance.
* @param selector A selector that provides a list of repository URIs.
* @param action An action to do for each repository.
* @param finalizationAction A final action to do for each repository,
* after all normal actions are done.
* @param credentials The credentials to use to operate on repositories.
* @param statusListener A listener that can be used to get information
* about the progress of the operations, and any
* messages that are emitted.
* @param workingDir The directory in which to do all git operations.
* @param strictFail Whether to fail instantly if any error occurs. If set
* to false, the program will continue even if actions
* fail for some repositories.
* @param cleanup Whether to perform cleanup after everything is done. This
* will remove the working directory once we're done.
*/
public DistribuGit(
RepositorySelector selector,
RepositoryAction action,
RepositoryAction finalizationAction,
GitCredentials credentials,
StatusListener statusListener,
Path workingDir,
@ -33,6 +78,7 @@ public class DistribuGit {
) {
this.selector = selector;
this.action = action;
this.finalizationAction = finalizationAction;
this.credentials = credentials;
this.statusListener = statusListener;
this.workingDir = workingDir;
@ -40,16 +86,35 @@ public class DistribuGit {
this.cleanup = cleanup;
}
public void doActions() throws IOException {
/**
* Performs the configured actions on the selected git repositories.
* @throws IOException If an error occurs that requires us to quit early.
* This is only thrown if {@link DistribuGit#strictFail} is true.
*/
public synchronized void doActions() throws IOException {
stepsComplete = 0;
if (Files.exists(workingDir)) {
try (var s = Files.list(workingDir)) {
if (s.findAny().isPresent()) throw new IOException("Working directory is not empty!");
}
}
Utils.delete(workingDir); // Delete the directory if it already exists.
Files.createDirectory(workingDir);
statusListener.messageReceived("Prepared temporary directory for repositories.");
List<String> repositoryURIs;
try {
List<String> repositoryURIs = selector.getURIs();
stepsTotal = 2 * repositoryURIs.size();
Map<String, Path> repoDirs = downloadRepositories(repositoryURIs);
applyActionToRepositories(repoDirs);
repositoryURIs = selector.getURIs();
} catch (Exception e) {
throw new IOException("Could not fetch repository URIs.", e);
}
try {
stepsTotal = 3 * repositoryURIs.size();
Map<String, Git> repos = downloadRepositories(repositoryURIs);
applyActionToRepositories(repos, action);
if (finalizationAction != null) {
applyActionToRepositories(repos, finalizationAction);
}
repos.values().forEach(Git::close);
} catch (Exception e) {
throw new IOException(e);
} finally {
@ -60,53 +125,93 @@ public class DistribuGit {
}
}
/**
* Runs the configured git actions on all selected repositories in an
* asynchronous manner.
* @return A future that completes when all actions are complete, or if an
* error occurs and the operation quits early.
*/
public CompletableFuture<Void> doActionsAsync() {
final CompletableFuture<Void> cf = new CompletableFuture<>();
ForkJoinPool.commonPool().submit(() -> {
try {
doActions();
cf.complete(null);
} catch (IOException e) {
cf.completeExceptionally(e);
}
});
return cf;
}
private void completeStep() {
stepsComplete++;
statusListener.progressUpdated(stepsComplete / (float) stepsTotal * 100f);
}
private Map<String, Path> downloadRepositories(List<String> uris) throws IOException {
Map<String, Path> repositoryDirs = new HashMap<>();
/**
* Downloads a set of repositories to working directories.
* @param uris The repositories to download.
* @return A map which maps each repository URI to its {@link Git} instance.
* @throws IOException If {@link DistribuGit#strictFail} is set to true,
* this will be thrown if an error occurs.
*/
private Map<String, Git> downloadRepositories(List<String> uris) throws IOException {
Map<String, Git> repositoryDirs = new HashMap<>();
int dirIdx = 1;
for (String repositoryURI : uris) {
Path repoDir = workingDir.resolve(Integer.toString(dirIdx++));
statusListener.messageReceived("Cloning repository " + repositoryURI + " to " + repoDir);
CloneCommand clone = Git.cloneRepository();
try {
statusListener.messageReceived("Cloning repository " + repositoryURI + " to " + repoDir);
CloneCommand clone = Git.cloneRepository();
credentials.addCredentials(clone);
clone.setDirectory(repoDir.toFile());
clone.setURI(repositoryURI);
try (var ignored = clone.call()) {
repositoryDirs.put(repositoryURI, repoDir);
} catch (Exception e) {
if (strictFail) {
throw new IOException(e);
} else {
repositoryDirs.put(repositoryURI, null);
e.printStackTrace();
}
}
} catch (Exception e) {
if (strictFail) {
throw new IOException(e);
}
statusListener.messageReceived("Could not add credentials to repository: " + e.getMessage());
e.printStackTrace();
// Skip the rest of the logic since this failed. Just go to the next repository.
completeStep();
continue;
}
clone.setDirectory(repoDir.toFile());
clone.setURI(repositoryURI);
try (var git = clone.call()) {
repositoryDirs.put(repositoryURI, git);
} catch (Exception e) {
if (strictFail) {
throw new IOException(e);
} else {
statusListener.messageReceived("Could not clone repository: " + e.getMessage());
repositoryDirs.put(repositoryURI, null);
e.printStackTrace();
}
}
completeStep();
}
return repositoryDirs;
}
private void applyActionToRepositories(Map<String, Path> repoDirs) throws IOException {
for (var entry : repoDirs.entrySet()) {
/**
* Applies an action to all git repositories in the given map.
* @param repositories A map which maps URIs to {@link Git} instances.
* @param action The action to apply to each repository.
* @throws IOException If {@link DistribuGit#strictFail} is set to true,
* this will be thrown if an error occurs.
*/
private void applyActionToRepositories(Map<String, Git> repositories, RepositoryAction action) throws IOException {
for (var entry : repositories.entrySet()) {
if (entry.getValue() != null) {
try (Git git = Git.open(entry.getValue().toFile())) {
try {
Git git = entry.getValue();
statusListener.messageReceived("Applying action to repository " + entry.getKey());
action.doAction(git);
} catch (Exception e) {
if (strictFail) {
throw new IOException(e);
}
statusListener.messageReceived("Action could not be applied to repository: " + e.getMessage());
e.printStackTrace();
}
} else {
@ -116,9 +221,14 @@ public class DistribuGit {
}
}
/**
* A builder class to help with constructing {@link DistribuGit} instances
* with a fluent method interface.
*/
public static class Builder {
private RepositorySelector selector;
private RepositoryAction action;
private RepositoryAction finalizationAction;
private GitCredentials credentials = cmd -> {};
private StatusListener statusListener = new StatusListener() {
@Override
@ -145,6 +255,11 @@ public class DistribuGit {
return this;
}
public Builder finalizationAction(RepositoryAction finalizationAction) {
this.finalizationAction = finalizationAction;
return this;
}
public Builder credentials(GitCredentials credentials) {
this.credentials = credentials;
return this;
@ -171,7 +286,10 @@ public class DistribuGit {
}
public DistribuGit build() {
return new DistribuGit(selector, action, credentials, statusListener, workingDir, strictFail, cleanup);
if (selector == null || action == null) {
throw new IllegalStateException("Cannot build an instance of DistribuGit without a selector or action.");
}
return new DistribuGit(selector, action, finalizationAction, credentials, statusListener, workingDir, strictFail, cleanup);
}
}
}

10
src/main/java/nl/andrewl/distribugit/GitCredentials.java
View File

@ -14,7 +14,17 @@ import org.eclipse.jgit.util.FS;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
/**
* Supplies credentials information to a git command, which is needed when
* interacting with private repositories, or when pushing changes to a
* repository.
*/
public interface GitCredentials {
/**
* Adds credentials information to the given command.
* @param gitCommand The command to add credentials to.
* @throws Exception If an error occurs while adding credentials.
*/
void addCredentials(TransportCommand<?, ?> gitCommand) throws Exception;
static GitCredentials ofUsernamePassword(String username, String password) {

8
src/main/java/nl/andrewl/distribugit/RepositoryAction.java
View File

@ -2,7 +2,15 @@ package nl.andrewl.distribugit;
import org.eclipse.jgit.api.Git;
/**
* An action that can be applied to a git repository.
*/
public interface RepositoryAction {
/**
* Performs the action on the given git repository.
* @param git A reference to the git repository.
* @throws Exception If an error occurs during the action.
*/
void doAction(Git git) throws Exception;
/**

5
src/main/java/nl/andrewl/distribugit/RepositorySelector.java
View File

@ -9,6 +9,11 @@ import java.util.List;
* Component which produces a list of repositories to operate on.
*/
public interface RepositorySelector {
/**
* Gets a list of repository URIs to operate on.
* @return A list of repository URIs.
* @throws Exception If an error occurs while fetching the URIs.
*/
List<String> getURIs() throws Exception;
static RepositorySelector fromCollection(Collection<String> uris) {

12
src/main/java/nl/andrewl/distribugit/StatusListener.java
View File

@ -1,6 +1,18 @@
package nl.andrewl.distribugit;
/**
* Listens for updates during {@link DistribuGit#doActions()}.
*/
public interface StatusListener {
/**
* Called when the operation's progress is updated.
* @param percentage The percentage (0 - 100) complete.
*/
void progressUpdated(float percentage);
/**
* Called when the DistribuGit operation emits a message.
* @param message The message that was emitted.
*/
void messageReceived(String message);
}

85
src/main/java/nl/andrewl/distribugit/cli/DistribuGitCommand.java Normal file
View File

@ -0,0 +1,85 @@
package nl.andrewl.distribugit.cli;
import nl.andrewl.distribugit.*;
import nl.andrewl.distribugit.selectors.GitHubSelectorBuilder;
import picocli.CommandLine;
import java.io.IOException;
import java.nio.file.Path;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@CommandLine.Command(name = "distribugit")
public class DistribuGitCommand implements Callable<Integer> {
private static final Pattern SELECTOR_EXPRESSION_PATTERN = Pattern.compile("([\\w-]+):(.+)");
private static final Pattern ORG_REPO_PREFIX_PATTERN = Pattern.compile("(.+)/(.+)");
@CommandLine.Option(names = {"-d", "--dir"}, description = "The working directory for DistribuGit", defaultValue = "./.distribugit_tmp")
private Path workingDir;
@CommandLine.Option(names = {"-s", "--selector"}, description = "The repository selector to use. Format: \"slug:content\"", required = true)
private String selectorExpression;
@CommandLine.Option(names = {"-a", "--action"}, description = "The command to run on each repository.", required = true)
private String actionCommand;
@CommandLine.Option(names = {"-fa", "--finalization-action"}, description = "A command to run on each repository after all normal actions.")
private String finalizationActionCommand;
@CommandLine.Option(names = {"-t", "--access-token"}, description = "The access token to use to perform operations.")
private String accessToken;
@CommandLine.Option(names = {"-sf", "--strict-fail"}, description = "Whether to preemptively fail if any error occurs.", defaultValue = "true")
private boolean strictFail;
@CommandLine.Option(names = {"-cl", "--cleanup"}, description = "Whether to remove all repository files when done.", defaultValue = "false")
private boolean cleanup;
@Override
public Integer call() throws Exception {
var builder = new DistribuGit.Builder()
.workingDir(workingDir)
.strictFail(strictFail)
.cleanup(cleanup)
.selector(parseSelectorExpression(selectorExpression))
.action(RepositoryAction.ofCommand(actionCommand.split("\\s+")));
if (finalizationActionCommand != null) {
builder.finalizationAction(RepositoryAction.ofCommand(finalizationActionCommand.split("\\s+")));
}
if (accessToken != null) {
builder.credentials(GitCredentials.ofUsernamePassword(accessToken, ""));
}
builder.statusListener(new StatusListener() {
@Override
public void progressUpdated(float percentage) {
System.out.printf("Progress: %.1f%%%n", percentage);
}
@Override
public void messageReceived(String message) {
System.out.println(message);
}
});
builder.build().doActions();
return 0;
}
private RepositorySelector parseSelectorExpression(String expr) throws IOException {
Matcher m = SELECTOR_EXPRESSION_PATTERN.matcher(expr);
if (!m.find()) throw new IllegalArgumentException("Invalid selector expression. Should be \"selector-type:expression\".");
String slug = m.group(1);
String content = m.group(2);
if (slug.equalsIgnoreCase("org-repo-prefix")) {
Matcher m1 = ORG_REPO_PREFIX_PATTERN.matcher(content);
if (!m1.find()) throw new IllegalArgumentException("Invalid content for org-repo-prefix select. Should be \"orgName/prefix\"");
return GitHubSelectorBuilder.fromPersonalAccessToken(accessToken).orgAndPrefix(m1.group(1), m1.group(2));
} else {
throw new IllegalArgumentException("Unsupported selector type: \"" + slug + "\".");
}
}
public static void main(String[] args) {
new CommandLine(new DistribuGitCommand()).execute(args);
}
}

10
src/main/java/nl/andrewl/distribugit/selectors/GitHubRepositorySelector.java Normal file
View File

@ -0,0 +1,10 @@
package nl.andrewl.distribugit.selectors;
import org.kohsuke.github.GHRepository;
import org.kohsuke.github.GitHub;
import java.util.List;
public interface GitHubRepositorySelector {
List<GHRepository> getRepos(GitHub gh) throws Exception;
}

55
src/main/java/nl/andrewl/distribugit/selectors/GitHubSelectorBuilder.java Normal file
View File

@ -0,0 +1,55 @@
package nl.andrewl.distribugit.selectors;
import nl.andrewl.distribugit.RepositorySelector;
import org.kohsuke.github.GHOrganization;
import org.kohsuke.github.GHRepository;
import org.kohsuke.github.GitHub;
import org.kohsuke.github.GitHubBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* A builder that can be used to construct {@link RepositorySelector} instances
* that fetch repositories from GitHub via its API.
*/
public record GitHubSelectorBuilder(GitHub gh) {
public static GitHubSelectorBuilder fromPersonalAccessToken(String token) throws IOException {
return fromPersonalAccessToken(token, null);
}
public static GitHubSelectorBuilder fromPersonalAccessToken(String token, String userOrOrgName) throws IOException {
return new GitHubSelectorBuilder(new GitHubBuilder().withOAuthToken(token, userOrOrgName).build());
}
/**
* Select repositories using an organization name, and a prefix to match
* against all repositories in that organization.
* @param orgName The name of the organization.
* @param prefix The prefix to use.
* @return A selector that selects matching repositories.
*/
public RepositorySelector orgAndPrefix(String orgName, String prefix) {
return () -> {
List<String> repoURIs = new ArrayList<>();
GHOrganization org = gh.getOrganization(orgName);
for (GHRepository repo : org.listRepositories()) {
if (repo.getName().startsWith(prefix)) {
repoURIs.add(repo.getHttpTransportUrl());
}
}
return repoURIs;
};
}
/**
* A custom selector that can be used to perform some operations using the
* GitHub API and return a list of repositories.
* @param selector The GitHub selector to use.
* @return A selector that selects GitHub repositories.
*/
public RepositorySelector custom(GitHubRepositorySelector selector) {
return () -> selector.getRepos(gh).stream().map(GHRepository::getHttpTransportUrl).toList();
}
}