diff --git a/.gitignore b/.gitignore index 4f216cd..f94758c 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ /ensync.iml *.xlsx# testfile.txt +*.log* diff --git a/src/main/java/com/olexyn/ensync/Flow.java b/src/main/java/com/olexyn/ensync/Flow.java index dce1f0e..7f184cb 100644 --- a/src/main/java/com/olexyn/ensync/Flow.java +++ b/src/main/java/com/olexyn/ensync/Flow.java @@ -3,65 +3,62 @@ package com.olexyn.ensync; import com.olexyn.ensync.artifacts.DataRoot; import com.olexyn.ensync.artifacts.StateFile; import com.olexyn.ensync.artifacts.SyncDirectory; -import com.olexyn.ensync.artifacts.SyncBundle; -import java.io.File; -import java.util.Map.Entry; import java.util.logging.Logger; - public class Flow implements Runnable { private static final Logger LOGGER = LogUtil.get(Flow.class); + public static final long POLLING_PAUSE = 400; - Tools tools = new Tools(); - - public long pollingPause = 200; - - - + /** + * + */ + @Override public void run() { while (true) { - synchronized(DataRoot.get()) { + synchronized(DataRoot.getSyncBundles()) { readOrMakeStateFile(); - for (Entry syncMapEntry : DataRoot.get().entrySet()) { - - for (Entry SDEntry : syncMapEntry.getValue().syncDirectories.entrySet()) { - - doSyncDirectory(SDEntry.getValue()); + DataRoot.getSyncBundles().forEach( + syncBundle -> { + var syncDirectories = syncBundle.getSyncDirectories(); + syncDirectories.forEach(this::doSyncDirectory); } - } + ); } + try { - System.out.println("Pausing... for " + pollingPause + "ms."); - Thread.sleep(pollingPause); + System.out.println("Pausing... for " + POLLING_PAUSE + "ms."); + Thread.sleep(POLLING_PAUSE); } catch (InterruptedException ignored) { } } } - - private void doSyncDirectory(SyncDirectory SD) { + /** + * + */ + private void doSyncDirectory(SyncDirectory sd) { LOGGER.info("READ"); - SD.readStateFromFS(); + sd.readStateFromFS(); - SD.listCreated = SD.makeListOfLocallyCreatedFiles(); - SD.listDeleted = SD.makeListOfLocallyDeletedFiles(); - SD.listModified = SD.makeListOfLocallyModifiedFiles(); - SD.doCreateOpsOnOtherSDs(); - SD.doDeleteOpsOnOtherSDs(); - SD.doModifyOpsOnOtherSDs(); + sd.makeListOfLocallyCreatedFiles(); + sd.makeListOfLocallyDeletedFiles(); + sd.makeListOfLocallyModifiedFiles(); - SD.writeStateFile(new StateFile(SD.path)); - } + sd.doCreateOpsOnOtherSDs(); + sd.doDeleteOpsOnOtherSDs(); + sd.doModifyOpsOnOtherSDs(); + sd.writeStateFile(new StateFile(sd.directoryPath)); + } /** * For every single SyncDirectory try to read it's StateFile.

@@ -70,13 +67,14 @@ public class Flow implements Runnable { private void readOrMakeStateFile() { DataRoot.get().values().forEach(syncBundle -> { for (var sd : syncBundle.syncDirectories.values()) { - var stateFile = new StateFile(sd.path); + var stateFile = new StateFile(sd.directoryPath); if (stateFile.exists()) { LOGGER.info("READ-STATE-FILE-" + sd.readStateFile()); } else { - sd.writeStateFile(new StateFile(sd.path)); + sd.writeStateFile(new StateFile(sd.directoryPath)); } } }); } + } diff --git a/src/main/java/com/olexyn/ensync/LogUtil.java b/src/main/java/com/olexyn/ensync/LogUtil.java index 0223066..70e76a0 100644 --- a/src/main/java/com/olexyn/ensync/LogUtil.java +++ b/src/main/java/com/olexyn/ensync/LogUtil.java @@ -23,7 +23,7 @@ public class LogUtil { System.setProperty("java.util.logging.SimpleFormatter.format", format); Logger logger = Logger.getLogger(c.getName()); try { - String dir = System.getProperty("user.dir") + "/log/main.log"; + String dir = System.getProperty("user.dir") + "/logs/main.log"; FileHandler fh = new FileHandler(dir, true); fh.setFormatter(new SimpleFormatter() { @Override diff --git a/src/main/java/com/olexyn/ensync/artifacts/DataRoot.java b/src/main/java/com/olexyn/ensync/artifacts/DataRoot.java index 0ea04e7..0b26b9a 100644 --- a/src/main/java/com/olexyn/ensync/artifacts/DataRoot.java +++ b/src/main/java/com/olexyn/ensync/artifacts/DataRoot.java @@ -1,5 +1,6 @@ package com.olexyn.ensync.artifacts; +import java.util.Collection; import java.util.HashMap; public class DataRoot { @@ -16,4 +17,9 @@ public class DataRoot { return mapOfSyncMaps; } + + public static Collection getSyncBundles() { + return get().values(); + } + } diff --git a/src/main/java/com/olexyn/ensync/artifacts/SyncBundle.java b/src/main/java/com/olexyn/ensync/artifacts/SyncBundle.java index 24dac1f..b8d710b 100644 --- a/src/main/java/com/olexyn/ensync/artifacts/SyncBundle.java +++ b/src/main/java/com/olexyn/ensync/artifacts/SyncBundle.java @@ -4,6 +4,7 @@ package com.olexyn.ensync.artifacts; import com.olexyn.ensync.Tools; import java.io.File; +import java.util.Collection; import java.util.HashMap; import java.util.Map; @@ -25,6 +26,10 @@ public class SyncBundle { this.name = name; } + public Collection getSyncDirectories() { + return syncDirectories.values(); + } + /** * Creates a new SyncDirectory.

* Adds the created SyncDirectory to this SyncBundle. diff --git a/src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java b/src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java index 24f645a..da1503c 100644 --- a/src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java +++ b/src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java @@ -1,7 +1,6 @@ package com.olexyn.ensync.artifacts; import com.olexyn.ensync.Execute; -import com.olexyn.ensync.Flow; import com.olexyn.ensync.LogUtil; import com.olexyn.ensync.Tools; @@ -11,10 +10,13 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Logger; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * A SyncDirectory is a singular occurrence of a directory in the filesystems. @@ -23,16 +25,12 @@ public class SyncDirectory { private static final Logger LOGGER = LogUtil.get(SyncDirectory.class); - private String flowState; - private SyncDirectory thisSD = this; - - private final SyncBundle syncMap; - public String path = null; + public String directoryPath; - public Map listCreated = new HashMap<>(); - public Map listDeleted = new HashMap<>(); - public Map listModified = new HashMap<>(); + public final Map listCreated = new HashMap<>(); + public final Map listDeleted = new HashMap<>(); + public final Map listModified = new HashMap<>(); Tools tools = new Tools(); @@ -43,9 +41,9 @@ public class SyncDirectory { * * @see SyncBundle */ - public SyncDirectory(String path, SyncBundle syncMap) { + public SyncDirectory(String directoryPath, SyncBundle syncMap) { - this.path = path; + this.directoryPath = directoryPath; this.syncMap = syncMap; } @@ -55,21 +53,12 @@ public class SyncDirectory { * Get the current state by using the `find` command. */ public Map readStateFromFS() { - //NOTE that the SFile().lastModifiedOld is not set here, so it is 0 by default. + //NOTE that the SyncFile().lastModifiedOld is not set here, so it is 0 by default. Map filemap = new HashMap<>(); - - Execute.TwoBr find = x.execute(new String[]{ - "find", - path - }); - - List pathList = tools.brToListString(find.output); - - for (String filePath : pathList) { - SyncFile file = new SyncFile(this, filePath); - - filemap.put(filePath, file); - } + getFiles() + .map(file -> new SyncFile(this, file.getAbsolutePath())) + .forEach(file -> filemap.put(file.getAbsolutePath(), file) + ); return filemap; } @@ -79,7 +68,7 @@ public class SyncDirectory { */ public Map readStateFile() { Map filemap = new HashMap<>(); - var stateFile = new StateFile(path); + var stateFile = new StateFile(directoryPath); List lines = tools.fileToLines(new File(stateFile.getPath())); for (String line : lines) { @@ -117,10 +106,10 @@ public class SyncDirectory { */ public Map makeListOfLocallyDeletedFiles() { - Map fromA = readStateFile(); - Map substractB = readStateFromFS(); + var fromA = readStateFile(); + var substractB = readStateFromFS(); - Map listDeleted = tools.mapMinus(fromA, substractB); + var listDeleted = tools.mapMinus(fromA, substractB); Map swap = new HashMap<>(); @@ -143,7 +132,8 @@ public class SyncDirectory { * Compare the OLD and NEW pools. * List is cleared and created each time. */ - public Map makeListOfLocallyModifiedFiles() { + public void makeListOfLocallyModifiedFiles() { + Map listModified = new HashMap<>(); @@ -164,7 +154,6 @@ public class SyncDirectory { listModified.put(freshFileKey, freshFile); } } - return listModified; } @@ -174,198 +163,88 @@ public class SyncDirectory { */ public void writeStateFile(StateFile stateFile) { List outputList = new ArrayList<>(); + getFiles().forEach( + file -> { + String relativePath = file.getAbsolutePath() + .replace(stateFile.getTargetPath(), ""); + outputList.add("" + file.lastModified() + " " + relativePath); + }); + tools.writeStringListToFile(stateFile.getPath(), outputList); + } + private Stream getFiles() { try { - Files.walk(Paths.get(path)) + return Files.walk(Paths.get(directoryPath)) .filter(Files::isRegularFile) .map(Path::toFile) - .filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME)) - .forEach(file -> { - String relativePath = file.getAbsolutePath() - .replace(stateFile.getTargetPath(), ""); - outputList.add("" + file.lastModified() + " " + relativePath); - }); + .filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME)); } catch (IOException e) { LOGGER.severe("Could walk the file tree : StateFile will be empty."); - } - - tools.writeStringListToFile(stateFile.getPath(), outputList); - } - - - private class Info { - - private String thisFilePath; - private String otherFilePath; - private String otherParentPath; - private File otherParentFile; - - - private Info(SyncDirectory thisSD, SyncFile sFile, SyncDirectory otherSD) { - // Example: - // syncDirectory /foo - // otherSyncDirectory /bar - // createdFile /foo/hello/created-file.gif - // relativePath /hello/created-file.gif - String relativePath = sFile.getPath().replace(thisSD.path, ""); - this.thisFilePath = sFile.getPath(); - this.otherFilePath = otherSD.path + relativePath; - File otherFile = new File(otherFilePath); - - this.otherParentPath = otherFile.getParent(); - this.otherParentFile = new File(otherParentPath); - - + return Stream.empty(); } } - public void doCreateOpsOnOtherSDs() { - - for (var entry : listCreated.entrySet()) { - SyncFile createdFile = entry.getValue(); - - for (var otherEntry : syncMap.syncDirectories.entrySet()) { - SyncDirectory otherSD = otherEntry.getValue(); - - if (this.equals(otherSD)) { continue; } - - Info info = new Info(this, createdFile, otherSD); - - writeFileIfNewer(info, createdFile, otherSD); + for (var createdFile : listCreated.values()) { + for (var otherFile : otherFiles(createdFile)) { + writeFileIfNewer(createdFile, otherFile); } } } - - /** - * - */ public void doDeleteOpsOnOtherSDs() { - - for (var entry : listDeleted.entrySet()) { - SyncFile deletedFile = entry.getValue(); - - for (var otherEntry : syncMap.syncDirectories.entrySet()) { - SyncDirectory otherSD = otherEntry.getValue(); - - if (this.equals(otherSD)) { continue; } - - Info info = new Info(thisSD, deletedFile, otherSD); - deleteFile(info, thisSD, deletedFile, otherSD); - - + for (var deletedFile : listDeleted.values()) { + for (var otherFile : otherFiles(deletedFile)) { + deleteFileIfNewer(deletedFile, otherFile); } } } - private void deleteFile(Info info, SyncDirectory thisSD, SyncFile thisFile, SyncDirectory otherSD) { - - SyncFile otherFile = new SyncFile(otherSD, otherSD.path + thisFile.relativePath); - - if (!otherFile.exists()) { return;} - - // if the otherFile was created with ensync it will have the == TimeModified. - long thisFileTimeModified = thisFile.getTimeModified(); - long otherFileTimeModified = otherFile.getTimeModified(); - - if (thisFile.getTimeModified() >= otherFile.getTimeModified()) { - List cmd = List.of("rm", "-r", info.otherFilePath); - x.execute(cmd); - } - } - - public void doModifyOpsOnOtherSDs() { - - for (var entry : listModified.entrySet()) { - SyncFile modifiedFile = entry.getValue(); - - for (var otherEntry : syncMap.syncDirectories.entrySet()) { - SyncDirectory otherSD = otherEntry.getValue(); - - if (this.equals(otherSD)) { continue; } - - Info info = new Info(this, modifiedFile, otherSD); - - writeFileIfNewer(info, modifiedFile, otherSD); + for (var modifiedFile : listModified.values()) { + for (var otherFile : otherFiles(modifiedFile)) { + writeFileIfNewer(modifiedFile, otherFile); } } } - /*** - * - * @param info - * @param thisFile - * @param otherSD - */ - private void writeFileIfNewer(Info info, SyncFile thisFile, SyncDirectory otherSD) { - - SyncFile otherFile = new SyncFile(otherSD, otherSD.path + thisFile.relativePath); - - if (otherFile.exists() && thisFile.getTimeModified() < otherFile.getTimeModified()) { return;} - - if (thisFile.isDirectory() && !otherFile.exists()) { - List cmd = List.of("mkdir", "-p", info.otherFilePath); - x.execute(cmd); - return; - } - - if (thisFile.isFile()) { + private Collection otherFiles(SyncFile thisFile) { + return syncMap.getSyncDirectories().stream() + .filter(sd -> !this.equals(sd)) + .map(thisFile::otherFile) + .collect(Collectors.toList()); + } - if (!info.otherParentFile.exists()) { - makeParentChain(otherFile, thisFile); + /** + * Delete other file if this file is newer. + */ + private void deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) { + if (!otherFile.exists()) { return; } + // if the otherFile was created with ensync it will have the == TimeModified. + if (thisFile.getTimeModified() >= otherFile.getTimeModified()) { + try { + Files.delete(Path.of(otherFile.getPath())); + } catch (IOException e) { + LOGGER.severe("Could not delete file."); } - - List cmd = List.of("cp", "-p", info.thisFilePath, info.otherFilePath); - x.execute(cmd); - copyModifDate(thisFile.getParentFile(), otherFile.getParentFile()); } } /** - * @param otherFile - * @param thisFile + * Overwrite other file if this file is newer. */ - private void makeParentChain(File otherFile, File thisFile) { - try { - File otherParent = new File(otherFile.getParent()); - File thisParent = new File(thisFile.getParent()); - - if (!otherParent.exists()) { - makeParentChain(otherParent, thisParent); - makeParentChain(otherFile, thisFile); - - } else if (thisFile.isDirectory()) { - - List cmd = List.of("mkdir", otherFile.getPath()); - x.execute(cmd); - - - cmd = List.of("stat", "--format", "%y", thisFile.getPath()); - - - String mDate = x.execute(cmd).output.readLine(); - - - cmd = List.of("touch", "-m", "--date=" + mDate, otherFile.getPath()); - String error = x.execute(cmd).error.readLine(); - int br = 0; - - + private void writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) { + if (otherFile.exists() && thisFile.isOlder(otherFile)) { return; } + if (thisFile.isFile()) { + try { + Files.copy( + Path.of(thisFile.getPath()), + Path.of(otherFile.getPath()) + ); + } catch (IOException e) { + LOGGER.severe("Could not copy file."); } - } catch (Exception ignored) {} - } - - - private void copyModifDate(File fromFile, File toFile) { - try { - List cmd = List.of("stat", "--format", "%y", fromFile.getPath()); - String mDate = x.execute(cmd).output.readLine(); - - cmd = List.of("touch", "-m", "--date=" + mDate, toFile.getPath()); - x.execute(cmd); - } catch (Exception ignored) {} + } } } diff --git a/src/main/java/com/olexyn/ensync/artifacts/SyncFile.java b/src/main/java/com/olexyn/ensync/artifacts/SyncFile.java index e69e504..be17a5c 100644 --- a/src/main/java/com/olexyn/ensync/artifacts/SyncFile.java +++ b/src/main/java/com/olexyn/ensync/artifacts/SyncFile.java @@ -1,7 +1,6 @@ package com.olexyn.ensync.artifacts; import java.io.File; -import java.util.Map; public class SyncFile extends File { @@ -16,7 +15,7 @@ public class SyncFile extends File { super(pathname); this.sd = sd; - relativePath = this.getPath().replace(sd.path, ""); + relativePath = this.getPath().replace(sd.directoryPath, ""); } public void setTimeModifiedFromStateFile(long value) { @@ -39,13 +38,26 @@ public class SyncFile extends File { * If a File never existed, it will have time = 0, and thus will always be overwritten. */ public long getTimeModified(){ - if (this.exists()){ + if (exists()) { return lastModified(); } - if (sd.readStateFile().get(this.getPath())!=null){ + if (sd.readStateFile().get(getPath()) != null) { return getTimeModifiedFromStateFile(); } - return 0; + return 0; } + + public boolean isNewer(SyncFile otherFile) { + return this.getTimeModified() >= otherFile.getTimeModified(); + } + + public boolean isOlder(SyncFile otherFile) { + return !isNewer(otherFile); + } + + public SyncFile otherFile(SyncDirectory otherSd) { + return new SyncFile(otherSd, otherSd.directoryPath + this.relativePath); + } + } diff --git a/src/test/test_cases.csv b/src/test/test_cases.csv new file mode 100644 index 0000000..4823fad --- /dev/null +++ b/src/test/test_cases.csv @@ -0,0 +1,10 @@ +Case +1 +2 +3 +4 +5 +6 +7 +8 +9 \ No newline at end of file