From 78fa7a5fedf12abf7f21c0719e73e02de4d1b1c0 Mon Sep 17 00:00:00 2001 From: io42630 Date: Mon, 4 Apr 2022 17:30:38 +0200 Subject: [PATCH] _ tests passed --- README.md | 10 +- doc/file-ops-covered-vs-missing.uxf | 4 +- doc/flow-n-instances.uxf | 8 +- doc/makeDiffLists.uxf | 6 +- pom.xml | 5 + src/main/java/com/olexyn/ensync/Flow.java | 62 +++--- src/main/java/com/olexyn/ensync/Tools.java | 17 +- .../olexyn/ensync/artifacts/Constants.java | 4 +- .../artifacts/{StateFile.kt => Record.kt} | 2 +- .../olexyn/ensync/artifacts/RecordFile.java | 17 ++ .../ensync/artifacts/SyncDirectory.java | 198 ++++++++++-------- .../com/olexyn/ensync/artifacts/SyncFile.java | 51 +++-- .../com/olexyn/ensync/files/FifteenTests.java | 157 +++++++------- .../java/com/olexyn/ensync/test-config.uxf | 4 +- 14 files changed, 301 insertions(+), 244 deletions(-) rename src/main/java/com/olexyn/ensync/artifacts/{StateFile.kt => Record.kt} (88%) create mode 100644 src/main/java/com/olexyn/ensync/artifacts/RecordFile.java diff --git a/README.md b/README.md index 6af0711..a5dda53 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ DataRoot a data root \_ SyncFile : a file on the FS. ``` -#### StateFile +#### Record * Used for tracking of file deletions. * Located in each `SyncDirectory\state.ensync` * Contains ` ` for each file in the SyncDirectory. @@ -67,13 +67,13 @@ src.com.olexyn.ensync. | Low level helper methods. - Reduce disk access. - Add error handling. (i.e. if a web-directory is not available) - Track files that were modified during the loop. - - currently `writeStateFile` just takes from `find` - - this means any changes made during the loop will be written to the `StateFile` - - and created files are tracked by comparing `StateFile` (=old state) and `State` (=new state). + - currently `writeRecord` just takes from `find` + - this means any changes made during the loop will be written to the `Record` + - and created files are tracked by comparing `Record` (=old state) and `State` (=new state). - because of this it will appear as if the file created while the loop was running was already there. - thus the creation of said file will not be replicated to the other directories. - - to solve this `writeStateFile` should take the old `State` + - to solve this `writeRecord` should take the old `State` and manually add every operation that was performed by the loop (!= user created file while the loop was running). - File is created in DirB - Sync creates the file in DirA diff --git a/doc/file-ops-covered-vs-missing.uxf b/doc/file-ops-covered-vs-missing.uxf index 7aca5e7..4bc75ee 100644 --- a/doc/file-ops-covered-vs-missing.uxf +++ b/doc/file-ops-covered-vs-missing.uxf @@ -327,7 +327,7 @@ nothing 40 cp if newer - try: time deletet = last time present in StateFile, else time deleted = 0 (~never existed) + try: time deletet = last time present in Record, else time deleted = 0 (~never existed) halign=left @@ -677,7 +677,7 @@ bg=red 720 30 - Deleted Files are tracked by their last existance in a StateFile. + Deleted Files are tracked by their last existance in a Record. diff --git a/doc/flow-n-instances.uxf b/doc/flow-n-instances.uxf index acbbfeb..bdca70d 100644 --- a/doc/flow-n-instances.uxf +++ b/doc/flow-n-instances.uxf @@ -56,7 +56,7 @@ bg=#81D4FA 50 read -StateFile +Record bg=#FFF59D halign=left style=wordwrap @@ -117,7 +117,7 @@ style=wordwrap 50 write -StateFile +Record bg=#FFF59D halign=left style=wordwrap @@ -259,7 +259,7 @@ transparency=0 50 [No -StateFile] +Record] @@ -315,7 +315,7 @@ StateFile] 100 50 - [StateFile + [Record exists] diff --git a/doc/makeDiffLists.uxf b/doc/makeDiffLists.uxf index d2ae852..534437a 100644 --- a/doc/makeDiffLists.uxf +++ b/doc/makeDiffLists.uxf @@ -50,7 +50,7 @@ group=1 130 60 - StateFile + Record bg=yellow halign=left group=1 @@ -105,7 +105,7 @@ group=2 130 30 - StateFile + Record bg=yellow halign=left group=2 @@ -234,7 +234,7 @@ group=3 130 30 - StateFile + Record bg=yellow halign=left group=3 diff --git a/pom.xml b/pom.xml index c38ee5f..a4334ca 100644 --- a/pom.xml +++ b/pom.xml @@ -49,6 +49,11 @@ kotlin-stdlib-jdk8 ${kotlin.version} + + commons-io + commons-io + 2.11.0 + diff --git a/src/main/java/com/olexyn/ensync/Flow.java b/src/main/java/com/olexyn/ensync/Flow.java index 9184c13..83a22b3 100644 --- a/src/main/java/com/olexyn/ensync/Flow.java +++ b/src/main/java/com/olexyn/ensync/Flow.java @@ -1,7 +1,7 @@ package com.olexyn.ensync; import com.olexyn.ensync.artifacts.DataRoot; -import com.olexyn.ensync.artifacts.StateFile; +import com.olexyn.ensync.artifacts.Record; import com.olexyn.ensync.artifacts.SyncDirectory; import java.util.concurrent.atomic.AtomicBoolean; @@ -12,10 +12,11 @@ public class Flow implements Runnable { private static final Logger LOGGER = LogUtil.get(Flow.class); - public static final long POLLING_PAUSE = 400; + public static final long POLLING_PAUSE = 100; private final AtomicBoolean running = new AtomicBoolean(false); public void start() { + LOGGER.info("START Flow."); Thread worker = new Thread(this); worker.start(); } @@ -28,19 +29,15 @@ public class Flow implements Runnable { public void run() { running.set(true); while (running.get()) { - synchronized(DataRoot.getSyncBundles()) { - - readOrMakeStateFile(); - + writeRecordIfMissing(); DataRoot.getSyncBundles().forEach( syncBundle -> { var syncDirectories = syncBundle.getSyncDirectories(); - syncDirectories.forEach(this::doSyncDirectory); + syncDirectories.forEach(this::sync); } ); } - try { LOGGER.info("Pausing... for " + POLLING_PAUSE + "ms."); Thread.sleep(POLLING_PAUSE); @@ -53,35 +50,36 @@ public class Flow implements Runnable { /** * */ - private void doSyncDirectory(SyncDirectory sd) { - LOGGER.info("DO SYNC DIRECTORY"); - sd.readFileSystem(); - - - sd.fillListOfLocallyCreatedFiles(); - sd.makeListOfLocallyDeletedFiles(); - sd.makeListOfLocallyModifiedFiles(); - - sd.doCreateOpsOnOtherSDs(); - sd.doDeleteOpsOnOtherSDs(); - sd.doModifyOpsOnOtherSDs(); - - - sd.writeStateFile(new StateFile(sd.directoryPath)); + private void sync(SyncDirectory sDir) { + LOGGER.info("DO SYNC " + sDir.directoryPath); + var listFileSystem = sDir.readFileSystem(); + LOGGER.info("# of files on FS: " + listFileSystem.size()); + var record = sDir.readRecord(); + LOGGER.info("# of files on Record: " + record.size()); + var listCreated = sDir.fillListOfLocallyCreatedFiles(listFileSystem, record); + LOGGER.info("# of files in Created: " + listCreated.size()); + var listDeleted = sDir.makeListOfLocallyDeletedFiles(listFileSystem, record); + LOGGER.info("# of files in Deleted: " + listDeleted.size()); + var listModified = sDir.makeListOfLocallyModifiedFiles(listFileSystem); + LOGGER.info("# of files in Modified: " + listModified.size()); + + sDir.doCreateOpsOnOtherSDs(listCreated); + sDir.doDeleteOpsOnOtherSDs(listDeleted); + sDir.doModifyOpsOnOtherSDs(listModified); + + sDir.writeRecord(new Record(sDir.directoryPath)); } /** - * For every single SyncDirectory try to read it's StateFile.

- * If the StateFile is missing, then create a StateFile. + * For every single SyncDirectory try to read it's Record.

+ * If the Record is missing, then create a Record. */ - private void readOrMakeStateFile() { + private void writeRecordIfMissing() { DataRoot.get().values().forEach(syncBundle -> { - for (var sd : syncBundle.syncDirectories.values()) { - var stateFile = new StateFile(sd.directoryPath); - if (stateFile.exists()) { - LOGGER.info("READ-STATE-FILE"); - } else { - sd.writeStateFile(new StateFile(sd.directoryPath)); + for (var sDir : syncBundle.syncDirectories.values()) { + var record = new Record(sDir.directoryPath); + if (!record.exists()) { + sDir.writeRecord(new Record(sDir.directoryPath)); } } }); diff --git a/src/main/java/com/olexyn/ensync/Tools.java b/src/main/java/com/olexyn/ensync/Tools.java index f24fefa..28c033d 100644 --- a/src/main/java/com/olexyn/ensync/Tools.java +++ b/src/main/java/com/olexyn/ensync/Tools.java @@ -7,8 +7,10 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; public class Tools { @@ -72,17 +74,12 @@ public class Tools { } - public Map mapMinus(Map fromA, Map substractB) { - - Map difference = new HashMap<>(); - for (Map.Entry entry : fromA.entrySet()) { - String key = entry.getKey(); - - if (fromA.containsKey(key) && !substractB.containsKey(key)) { - SyncFile file = fromA.get(key); - difference.put(key, file); + public Set setMinus(Set fromA, Set subtractB) { + Set difference = new HashSet<>(); + for (var key : fromA) { + if (fromA.contains(key) && !subtractB.contains(key)) { + difference.add(key); } - } return difference; } diff --git a/src/main/java/com/olexyn/ensync/artifacts/Constants.java b/src/main/java/com/olexyn/ensync/artifacts/Constants.java index 3db0966..c6edff8 100644 --- a/src/main/java/com/olexyn/ensync/artifacts/Constants.java +++ b/src/main/java/com/olexyn/ensync/artifacts/Constants.java @@ -2,6 +2,8 @@ package com.olexyn.ensync.artifacts; public interface Constants { - String STATE_FILE_NAME = "state.ensync"; + String STATE_FILE_NAME = "record.ensync"; + String SPACE = " "; + String EMPTY = ""; } diff --git a/src/main/java/com/olexyn/ensync/artifacts/StateFile.kt b/src/main/java/com/olexyn/ensync/artifacts/Record.kt similarity index 88% rename from src/main/java/com/olexyn/ensync/artifacts/StateFile.kt rename to src/main/java/com/olexyn/ensync/artifacts/Record.kt index ee3ef3d..6938d17 100644 --- a/src/main/java/com/olexyn/ensync/artifacts/StateFile.kt +++ b/src/main/java/com/olexyn/ensync/artifacts/Record.kt @@ -3,7 +3,7 @@ package com.olexyn.ensync.artifacts import java.io.File import java.nio.file.Path -class StateFile(val targetPath: Path) { +class Record(val targetPath: Path) { fun getPath(): Path { return targetPath.resolve(Constants.STATE_FILE_NAME) diff --git a/src/main/java/com/olexyn/ensync/artifacts/RecordFile.java b/src/main/java/com/olexyn/ensync/artifacts/RecordFile.java new file mode 100644 index 0000000..337b367 --- /dev/null +++ b/src/main/java/com/olexyn/ensync/artifacts/RecordFile.java @@ -0,0 +1,17 @@ +package com.olexyn.ensync.artifacts; + +public class RecordFile extends SyncFile { + + // Very IMPORTANT field. Allows to store lastModified as it is stored in the Record. + public long timeModifiedFromRecord = 0; + + public RecordFile(SyncDirectory sDir, String absolutePath) { + super(sDir, absolutePath); + } + + public void setTimeModifiedFromRecord(long value) { + timeModifiedFromRecord = value; + } + + +} diff --git a/src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java b/src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java index 70ca9b7..20b6d06 100644 --- a/src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java +++ b/src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java @@ -1,40 +1,43 @@ package com.olexyn.ensync.artifacts; -import com.olexyn.ensync.Execute; import com.olexyn.ensync.LogUtil; import com.olexyn.ensync.Tools; import java.io.File; +import java.io.FileInputStream; import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.CopyOption; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +import static com.olexyn.ensync.artifacts.Constants.EMPTY; +import static com.olexyn.ensync.artifacts.Constants.SPACE; + /** * A SyncDirectory is a singular occurrence of a directory in the filesystems. */ public class SyncDirectory { private static final Logger LOGGER = LogUtil.get(SyncDirectory.class); - private final SyncBundle syncMap; public Path directoryPath; - - public final Map listCreated = new HashMap<>(); - public final Map listDeleted = new HashMap<>(); - public final Map listModified = new HashMap<>(); - - + public Map listCreated = new HashMap<>(); + public Map listDeleted = new HashMap<>(); + public Map listModified = new HashMap<>(); Tools tools = new Tools(); - Execute x = new Execute(); /** * Create a SyncDirectory from realPath. @@ -42,10 +45,8 @@ public class SyncDirectory { * @see SyncBundle */ public SyncDirectory(Path directoryPath, SyncBundle syncMap) { - this.directoryPath = directoryPath; this.syncMap = syncMap; - } @@ -53,7 +54,6 @@ public class SyncDirectory { * Read the current state of the file system. */ public Map readFileSystem() { - //NOTE that the SyncFile().lastModifiedOld is not set here, so it is 0 by default. return getFiles() .map(file -> new SyncFile(this, file.getAbsolutePath())) .collect(Collectors.toMap( @@ -62,24 +62,22 @@ public class SyncDirectory { )); } - /** - * READ the contents of StateFile to Map. + * READ the contents of Record to Map. */ - public Map readStateFile() { - Map filemap = new HashMap<>(); - var stateFile = new StateFile(directoryPath); - List lines = tools.fileToLines(stateFile.getPath().toFile()); + public Map readRecord() { + Map filemap = new HashMap<>(); + var record = new Record(directoryPath); + List lines = tools.fileToLines(record.getPath().toFile()); for (String line : lines) { - // this is a predefined format: "modification-time path" - String modTimeString = line.split(" ")[0]; - long modTime = Long.parseLong(modTimeString); - - String sFilePath = line.replace(modTimeString + " ", ""); - SyncFile sfile = new SyncFile(this, sFilePath); + // this is a predefined format: " " + var lineArr = line.split(SPACE); + long modTime = Long.parseLong(lineArr[0]); + String sFilePath = lineArr[1]; + RecordFile sfile = new RecordFile(this, sFilePath); - sfile.setTimeModifiedFromStateFile(modTime); + sfile.setTimeModifiedFromRecord(modTime); filemap.put(sFilePath, sfile); } @@ -91,58 +89,48 @@ public class SyncDirectory { * Compare the OLD and NEW pools. * List is cleared and created each time. */ - public void fillListOfLocallyCreatedFiles() { - listCreated.clear(); - var fromA = readFileSystem(); - var substractB = readStateFile(); - listCreated.putAll(tools.mapMinus(fromA, substractB)); + public Map fillListOfLocallyCreatedFiles(Map listFileSystem, Map record) { + var listCreated = tools.setMinus(listFileSystem.keySet(), record.keySet()); + return listCreated.stream().collect(Collectors.toMap(key -> key, listFileSystem::get)); } - /** * Compare the OLD and NEW pools. * List is cleared and created each time. */ - public void makeListOfLocallyDeletedFiles() { - listDeleted.clear(); - var fromA = readStateFile(); - var substractB = readFileSystem(); - var listDeleted = tools.mapMinus(fromA, substractB); - Map swap = new HashMap<>(); - for (var entry : listDeleted.entrySet()) { - String key = entry.getKey(); - String parentKey = entry.getValue().getParent(); - if (listDeleted.containsKey(parentKey) || swap.containsKey(parentKey)) { - swap.put(key, listDeleted.get(key)); - } - } - listDeleted.putAll(tools.mapMinus(listDeleted, swap)); + public Map makeListOfLocallyDeletedFiles(Map listFileSystem, Map record) { + var listDeleted = tools.setMinus(record.keySet(), listFileSystem.keySet()); + return listDeleted.stream().collect(Collectors.toMap(key -> key, record::get)); } - /** * Compare the OLD and NEW pools. * List is cleared and created each time. */ - public void makeListOfLocallyModifiedFiles() { - listModified.clear(); - - Map stateFileMap = readStateFile(); - - for (var freshFileEntry : readFileSystem().entrySet()) { - - String freshFileKey = freshFileEntry.getKey(); - SyncFile freshFile = freshFileEntry.getValue(); - - if (freshFile.isDirectory()) { continue;} // no need to modify Directories, the Filesystem will do that, if a File changed. - - // If KEY exists in OLD , thus FILE was NOT created. - boolean oldFileExists = stateFileMap.containsKey(freshFileKey); - boolean fileIsFresher = freshFile.getTimeModified() > freshFile.getTimeModifiedFromStateFile(); - - if (oldFileExists && fileIsFresher) { - listModified.put(freshFileKey, freshFile); + public Map makeListOfLocallyModifiedFiles(Map listFileSystem) { + + return listFileSystem.entrySet().stream().filter( + fileEntry -> { + String fileKey = fileEntry.getKey(); + SyncFile file = fileEntry.getValue(); + if (file.isDirectory()) { return false; } // no need to modify Directories, the Filesystem will do that, if a File changed. + boolean isKnown = readRecord().containsKey(fileKey); // If KEY exists in OLD , thus FILE was NOT created. + boolean isModified = file.lastModified() > file.lastModifiedFromRecord(); + return isKnown && isModified; } + ).collect(Collectors.toMap(Entry::getKey, Entry::getValue)); + } + + private String getMd5(Path path) { + try (var fos = new FileInputStream(path.toFile())) { + var m = MessageDigest.getInstance("MD5"); + byte[] data = fos.readAllBytes(); + m.update(data, 0, data.length); + var i = new BigInteger(1, m.digest()); + return String.format("%1$032X", i); + } catch (NoSuchAlgorithmException | IOException e) { + LOGGER.info("File not found."); + return null; } } @@ -150,15 +138,21 @@ public class SyncDirectory { * QUERY state of the filesystem at realPath. * WRITE the state of the filesystem to file. */ - public void writeStateFile(StateFile stateFile) { + public void writeRecord(Record record) { List outputList = new ArrayList<>(); getFiles().forEach( file -> { String relativePath = file.getAbsolutePath() - .replace(stateFile.getTargetPath().toString(), ""); - outputList.add("" + file.lastModified() + " " + relativePath); + .replace(record.getTargetPath().toString(), EMPTY); + var line = String.join( + SPACE, + String.valueOf(file.lastModified()), + relativePath + ); + outputList.add(line); }); - tools.writeStringListToFile(stateFile.getPath().toString(), outputList); + LOGGER.info("Writing " + outputList.size() + " files to " + record.getPath()); + tools.writeStringListToFile(record.getPath().toString(), outputList); } private Stream getFiles() { @@ -168,12 +162,12 @@ public class SyncDirectory { .map(Path::toFile) .filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME)); } catch (IOException e) { - LOGGER.severe("Could walk the file tree : StateFile will be empty."); + LOGGER.severe("Could walk the file tree : Record will be empty."); return Stream.empty(); } } - public void doCreateOpsOnOtherSDs() { + public void doCreateOpsOnOtherSDs(Map listCreated) { for (var createdFile : listCreated.values()) { for (var otherFile : otherFiles(createdFile)) { writeFileIfNewer(createdFile, otherFile); @@ -181,7 +175,7 @@ public class SyncDirectory { } } - public void doDeleteOpsOnOtherSDs() { + public void doDeleteOpsOnOtherSDs(Map listDeleted) { for (var deletedFile : listDeleted.values()) { for (var otherFile : otherFiles(deletedFile)) { deleteFileIfNewer(deletedFile, otherFile); @@ -189,7 +183,7 @@ public class SyncDirectory { } } - public void doModifyOpsOnOtherSDs() { + public void doModifyOpsOnOtherSDs(Map listModified) { for (var modifiedFile : listModified.values()) { for (var otherFile : otherFiles(modifiedFile)) { writeFileIfNewer(modifiedFile, otherFile); @@ -210,11 +204,12 @@ public class SyncDirectory { private void deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) { if (!otherFile.exists()) { return; } // if the otherFile was created with ensync it will have the == TimeModified. - if (thisFile.getTimeModified() >= otherFile.getTimeModified()) { + if (thisFile.lastModified() >= otherFile.lastModified()) { try { - Files.delete(Path.of(otherFile.getPath())); + Files.delete(otherFile.toPath()); + LOGGER.info("Deleted: " + otherFile.toPath()); } catch (IOException e) { - LOGGER.severe("Could not delete file."); + LOGGER.severe("Could not delete: " + otherFile.toPath()); } } } @@ -223,18 +218,49 @@ public class SyncDirectory { * Overwrite other file if this file is newer. */ private void writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) { - if (otherFile.exists() && thisFile.isOlder(otherFile)) { return; } - if (thisFile.isFile()) { - try { - Files.copy( - Path.of(thisFile.getPath()), - Path.of(otherFile.getPath()) - ); - otherFile.setLastModified(thisFile.lastModified()); - } catch (IOException e) { - LOGGER.severe("Could not copy file."); + LOGGER.info("Write from: " + thisFile.toPath()); + LOGGER.info(" to: " + otherFile.toPath()); + if (!thisFile.isFile()) { return; } + if (otherFile.exists()) { + var thisMd5 = getMd5(thisFile.toPath()); + var otherMd5 = getMd5(otherFile.toPath()); + if (thisMd5 == null || otherMd5 == null) { return; } + if (thisMd5.equals(otherMd5)) { + dropAge(thisFile, otherFile); + return; + } else if (thisFile.isOlder(otherFile)) { + LOGGER.info("Did not override due to target being newer."); + return; } } + copyFile(thisFile, otherFile); + } + + private void dropAge(SyncFile thisFile, SyncFile otherFile) { + if (thisFile.isOlder(otherFile)) { + otherFile.setLastModified(thisFile.lastModified()); + LOGGER.info("Dropped age of OTHER: " + otherFile.toPath() + " to: " + otherFile.lastModified()); + } else { + thisFile.setLastModified(otherFile.lastModified()); + LOGGER.info("Dropped age of THIS: " + thisFile.toPath() + " to: " + thisFile.lastModified()); + } + } + + private void copyFile(SyncFile thisFile, SyncFile otherFile) { + try { + Files.copy( + Path.of(thisFile.getPath()), + Path.of(otherFile.getPath()), + StandardCopyOption.REPLACE_EXISTING, + StandardCopyOption.COPY_ATTRIBUTES + ); + LOGGER.info("Copied from: " + thisFile.toPath()); + LOGGER.info(" to: " + otherFile.toPath()); + } catch (IOException e) { + LOGGER.severe("Could not copy file from: " + thisFile.toPath()); + LOGGER.severe(" to: " + otherFile.toPath()); + e.printStackTrace(); + } } } diff --git a/src/main/java/com/olexyn/ensync/artifacts/SyncFile.java b/src/main/java/com/olexyn/ensync/artifacts/SyncFile.java index 893e96e..c503955 100644 --- a/src/main/java/com/olexyn/ensync/artifacts/SyncFile.java +++ b/src/main/java/com/olexyn/ensync/artifacts/SyncFile.java @@ -5,61 +5,60 @@ import com.olexyn.ensync.LogUtil; import java.io.File; import java.util.logging.Logger; +import static com.olexyn.ensync.artifacts.Constants.EMPTY; + public class SyncFile extends File { private static final Logger LOGGER = LogUtil.get(SyncFile.class); - // Very IMPORTANT field. Allows to store lastModified as it is stored in the StateFile. - public long timeModifiedFromStateFile = 0; - private final String relativePath; - private final SyncDirectory sd; - public SyncFile(SyncDirectory sd, String absolutePath) { + private final String relativePath; + private final SyncDirectory sDir; + public SyncFile(SyncDirectory sDir, String absolutePath) { super(absolutePath); - this.sd = sd; - relativePath = this.getPath().replace(sd.directoryPath.toString(), ""); + this.sDir = sDir; + relativePath = this.getPath().replace(sDir.directoryPath.toString(), EMPTY); } public String getRelativePath() { return relativePath; } - public void setTimeModifiedFromStateFile(long value) { - timeModifiedFromStateFile = value; - } - public long getTimeModifiedFromStateFile() { - SyncFile record = sd.readStateFile().get(getRelativePath()); + + public long lastModifiedFromRecord() { + RecordFile record = getFromRecord(); if (record == null) { - LOGGER.severe("Did not find record for file in StateFile. Setting modifiedDate to MIN, thus 0"); - return 0; + LOGGER.severe("Did not find record for " + this.toPath() + " in Record."); + LOGGER.severe("Returning -1 (never existed)."); + return -1; } - return record.timeModifiedFromStateFile; + return record.timeModifiedFromRecord; + } + + public RecordFile getFromRecord() { + return sDir.readRecord().get(getRelativePath()); } /** * If File exists on Disk get the TimeModified from there. - * Else try to read it from StateFile. + * Else try to read it from Record. * Else return 0 ( = oldest possible time - a value of 0 can be seen as equal to "never existed"). * EXAMPLES: * If a File was deleted, then the time will be taken from statefile. * If a File never existed, it will have time = 0, and thus will always be overwritten. */ - public long getTimeModified(){ - if (exists()) { - return lastModified(); - } - - if (sd.readStateFile().get(getPath()) != null) { - return getTimeModifiedFromStateFile(); - } - return 0; + @Override + public long lastModified(){ + if (exists()) { return super.lastModified(); } + return lastModifiedFromRecord(); } public boolean isNewer(SyncFile otherFile) { - return this.getTimeModified() >= otherFile.getTimeModified(); + LOGGER.info("" + this.lastModified() + " >= " + otherFile.lastModified()); + return this.lastModified() >= otherFile.lastModified(); } public boolean isOlder(SyncFile otherFile) { diff --git a/src/test/java/com/olexyn/ensync/files/FifteenTests.java b/src/test/java/com/olexyn/ensync/files/FifteenTests.java index 9869ed5..c7d4517 100644 --- a/src/test/java/com/olexyn/ensync/files/FifteenTests.java +++ b/src/test/java/com/olexyn/ensync/files/FifteenTests.java @@ -21,6 +21,8 @@ import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; +import org.apache.commons.io.FileUtils; + /** * Perform the 15 test cases in TestCases.xlsx. @@ -35,17 +37,12 @@ public class FifteenTests { final private static Tools tools = new Tools(); - private final static long FILE_OPS_PAUSE = 800; - private final static long WAIT_BEFORE_ASSERT = 4000; - - - private final static Execute x = new Execute(); + private final static long M1000 = 600; private static final Path TEMP_DIR = Path.of(System.getProperty("user.dir") + "/src/test/temp"); - private static final String RESOURCES_DIR = System.getProperty("user.dir") + "/src/test/resources"; DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; - private static final Path A_DIR = TEMP_DIR.resolve("a"); - private static final Path B_DIR = TEMP_DIR.resolve("b"); + private static final Path A_DIR = TEMP_DIR.resolve("AAA"); + private static final Path B_DIR = TEMP_DIR.resolve("BBB"); private final TestFile aFile = new TestFile(A_DIR + "/testfile.txt"); private final TestFile bFile = new TestFile(B_DIR + "/testfile.txt"); @@ -55,20 +52,22 @@ public class FifteenTests { try { stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " CREATED"); tools.writeStringListToFile(file.getAbsolutePath(), stringList); - Thread.sleep(FILE_OPS_PAUSE); + LOGGER.info("TEST CREATE: " + file.toPath()); + Thread.sleep(M1000); } catch (InterruptedException e) { System.out.println(""); } return stringList; } - private List updateFile(File file) { + private List modifyFile(File file) { List stringList = new ArrayList<>(); try { stringList.addAll(tools.fileToLines(file)); - stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " UPDATED"); + stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " MODIFIED"); tools.writeStringListToFile(file.getAbsolutePath(), stringList); - Thread.sleep(FILE_OPS_PAUSE); + LOGGER.info("TEST MODIFY: " + file.toPath()); + Thread.sleep(M1000); } catch (InterruptedException e) { System.out.println(""); } @@ -79,40 +78,20 @@ public class FifteenTests { private static void deleteFile(File file) { try { Files.delete(file.toPath()); - Thread.sleep(FILE_OPS_PAUSE); + LOGGER.info("TEST DELETE: " + file.toPath()); + Thread.sleep(M1000); } catch (IOException | InterruptedException e) { - LOGGER.severe("Could not delete file."); - } - } - - private static void deleteRec(Path path) { - var file = path.toFile(); - if (!file.exists()) { - return; - } - if (file.isDirectory()) { - try { - Files.walk(path) - .filter(subPath -> !subPath.equals(path)) - .forEach(FifteenTests::deleteRec); - } catch (IOException e) { - LOGGER.severe("Could not walk path."); - } - } - try { - Files.delete(path); - } catch (IOException e) { - LOGGER.severe("Could not delete file."); + LOGGER.severe("Could not delete file." + file.toPath()); } } private void cleanDirs(Path... dirs) { for (var dir : dirs) { - deleteRec(dir); try { + FileUtils.deleteDirectory(dir.toFile()); Files.createDirectory(dir); } catch (IOException e) { - LOGGER.severe("Could not clear dirs."); + LOGGER.severe("Could not clear dirs. " + dir + e.getMessage()); } } } @@ -120,9 +99,9 @@ public class FifteenTests { SyncBundle syncBundle; - public static void waitBeforeAssert() { + public static void waitX(long time) { try { - Thread.sleep(WAIT_BEFORE_ASSERT); + Thread.sleep(time); } catch (InterruptedException ignored) {} } @@ -133,123 +112,157 @@ public class FifteenTests { syncBundle.addDirectory(B_DIR); cleanDirs(A_DIR, B_DIR); DataRoot.get().put(syncBundle.name, syncBundle); - FLOW.start(); } @After public void reset() { FLOW.stop(); + waitX(M1000); cleanDirs(A_DIR, B_DIR); } @Test - public void one() { + public void test1() { createFile(aFile); + waitX(M1000); + FLOW.start(); + waitX(M1000); deleteFile(aFile); - waitBeforeAssert(); + waitX(M1000); Assert.assertFalse(aFile.exists()); Assert.assertFalse(bFile.exists()); } @Test - public void three() { + public void test2() { createFile(aFile); createFile(bFile); + waitX(M1000); + FLOW.start(); + waitX(M1000); deleteFile(aFile); deleteFile(bFile); - waitBeforeAssert(); + waitX(M1000); Assert.assertFalse(aFile.exists()); Assert.assertFalse(bFile.exists()); } @Test - public void four() { + public void test3() { createFile(aFile); + waitX(M1000); + FLOW.start(); + waitX(M1000); deleteFile(aFile); var bContent = createFile(bFile); - waitBeforeAssert(); + waitX(M1000); Assert.assertEquals(bContent, aFile.readContent()); } @Test - public void five() { + public void test4() { createFile(aFile); createFile(bFile); + waitX(M1000); + FLOW.start(); + waitX(M1000); deleteFile(aFile); - var bContent = updateFile(bFile); - waitBeforeAssert(); + var bContent = modifyFile(bFile); + waitX(M1000); Assert.assertEquals(bContent, aFile.readContent()); } @Test - public void six() { + public void test5() { + FLOW.start(); + waitX(M1000); var aContent = createFile(aFile); - waitBeforeAssert(); + waitX(M1000); Assert.assertEquals(aContent, bFile.readContent()); } @Test - public void eight() { - createFile(aFile); + public void test6() { createFile(bFile); + waitX(M1000); + FLOW.start(); + waitX(M1000); + createFile(aFile); deleteFile(bFile); - waitBeforeAssert(); + waitX(M1000); Assert.assertFalse(aFile.exists()); Assert.assertFalse(bFile.exists()); } @Test - public void nine() { + public void test7() { + FLOW.start(); + waitX(M1000); createFile(aFile); var bContent = createFile(bFile); - waitBeforeAssert(); + waitX(M1000); Assert.assertEquals(bContent, aFile.readContent()); } @Test - public void ten() { + public void test8() { createFile(bFile); + waitX(M1000); + FLOW.start(); + waitX(M1000); createFile(aFile); - var bContent = updateFile(bFile); - waitBeforeAssert(); + var bContent = modifyFile(bFile); + waitX(M1000); Assert.assertEquals(bContent, aFile.readContent()); } @Test - public void eleven() { + public void test9() { createFile(aFile); - var aContent = updateFile(aFile); - waitBeforeAssert(); + waitX(M1000); + FLOW.start(); + waitX(M1000); + var aContent = modifyFile(aFile); + waitX(M1000); Assert.assertEquals(aContent, bFile.readContent()); } @Test - public void thirteen() { + public void test10() { createFile(aFile); createFile(bFile); - updateFile(aFile); + waitX(M1000); + FLOW.start(); + waitX(M1000); + modifyFile(aFile); deleteFile(bFile); - waitBeforeAssert(); + waitX(M1000); Assert.assertFalse(aFile.exists()); Assert.assertFalse(bFile.exists()); } @Test - public void fourteen() { + public void test11() { createFile(aFile); - updateFile(aFile); + waitX(M1000); + FLOW.start(); + waitX(M1000); + modifyFile(aFile); var bContent = createFile(bFile); - waitBeforeAssert(); + waitX(M1000); Assert.assertEquals(bContent, aFile.readContent()); } @Test - public void fifteen() { + public void test12() { createFile(aFile); createFile(bFile); - updateFile(aFile); - var bContent = updateFile(bFile); - waitBeforeAssert(); + waitX(M1000); + FLOW.start(); + waitX(M1000); + modifyFile(aFile); + var bContent = modifyFile(bFile); + waitX(M1000); Assert.assertEquals(bContent, aFile.readContent()); } diff --git a/src/test/java/com/olexyn/ensync/test-config.uxf b/src/test/java/com/olexyn/ensync/test-config.uxf index 7367943..30a4f19 100644 --- a/src/test/java/com/olexyn/ensync/test-config.uxf +++ b/src/test/java/com/olexyn/ensync/test-config.uxf @@ -327,7 +327,7 @@ nothing 40 cp if newer - try: time deletet = last time present in StateFile, else time deleted = 0 (~never existed) + try: time deletet = last time present in Record, else time deleted = 0 (~never existed) halign=left @@ -677,7 +677,7 @@ bg=red 720 30 - Deleted Files are tracked by their last existance in a StateFile. + Deleted Files are tracked by their last existance in a Record.