_ tests passed

pull/1/head
io42630 3 years ago
parent 8c6f296272
commit 78fa7a5fed

@ -27,7 +27,7 @@ DataRoot a data root
\_ SyncFile : a file on the FS. \_ SyncFile : a file on the FS.
``` ```
#### StateFile #### Record
* Used for tracking of file deletions. * Used for tracking of file deletions.
* Located in each `SyncDirectory\state.ensync` * Located in each `SyncDirectory\state.ensync`
* Contains `<last edited> <relative file path>` for each file in the SyncDirectory. * Contains `<last edited> <relative file path>` for each file in the SyncDirectory.
@ -67,13 +67,13 @@ src.com.olexyn.ensync. | Low level helper methods.
- Reduce disk access. - Reduce disk access.
- Add error handling. (i.e. if a web-directory is not available) - Add error handling. (i.e. if a web-directory is not available)
- Track files that were modified during the loop. - Track files that were modified during the loop.
- currently `writeStateFile` just takes from `find` - currently `writeRecord` just takes from `find`
- this means any changes made during the loop will be written to the `StateFile` - this means any changes made during the loop will be written to the `Record`
- and created files are tracked by comparing `StateFile` (=old state) and `State` (=new state). - and created files are tracked by comparing `Record` (=old state) and `State` (=new state).
- because of this it will appear as if the file created while the loop was running - because of this it will appear as if the file created while the loop was running
was already there. was already there.
- thus the creation of said file will not be replicated to the other directories. - thus the creation of said file will not be replicated to the other directories.
- to solve this `writeStateFile` should take the old `State` - to solve this `writeRecord` should take the old `State`
and manually add every operation that was performed by the loop (!= user created file while the loop was running). and manually add every operation that was performed by the loop (!= user created file while the loop was running).
- File is created in DirB - File is created in DirB
- Sync creates the file in DirA - Sync creates the file in DirA

@ -327,7 +327,7 @@ nothing</panel_attributes>
<h>40</h> <h>40</h>
</coordinates> </coordinates>
<panel_attributes> cp if newer <panel_attributes> cp if newer
try: time deletet = last time present in StateFile, else time deleted = 0 (~never existed) try: time deletet = last time present in Record, else time deleted = 0 (~never existed)
halign=left</panel_attributes> halign=left</panel_attributes>
<additional_attributes/> <additional_attributes/>
</element> </element>
@ -677,7 +677,7 @@ bg=red</panel_attributes>
<w>720</w> <w>720</w>
<h>30</h> <h>30</h>
</coordinates> </coordinates>
<panel_attributes>Deleted Files are tracked by their last existance in a StateFile.</panel_attributes> <panel_attributes>Deleted Files are tracked by their last existance in a Record.</panel_attributes>
<additional_attributes/> <additional_attributes/>
</element> </element>
<element> <element>

@ -56,7 +56,7 @@ bg=#81D4FA</panel_attributes>
<h>50</h> <h>50</h>
</coordinates> </coordinates>
<panel_attributes>read <panel_attributes>read
StateFile Record
bg=#FFF59D bg=#FFF59D
halign=left halign=left
style=wordwrap style=wordwrap
@ -117,7 +117,7 @@ style=wordwrap</panel_attributes>
<h>50</h> <h>50</h>
</coordinates> </coordinates>
<panel_attributes>write <panel_attributes>write
StateFile Record
bg=#FFF59D bg=#FFF59D
halign=left halign=left
style=wordwrap style=wordwrap
@ -259,7 +259,7 @@ transparency=0</panel_attributes>
<h>50</h> <h>50</h>
</coordinates> </coordinates>
<panel_attributes>[No <panel_attributes>[No
StateFile]</panel_attributes> Record]</panel_attributes>
<additional_attributes/> <additional_attributes/>
</element> </element>
<element> <element>
@ -315,7 +315,7 @@ StateFile]</panel_attributes>
<w>100</w> <w>100</w>
<h>50</h> <h>50</h>
</coordinates> </coordinates>
<panel_attributes>[StateFile <panel_attributes>[Record
exists]</panel_attributes> exists]</panel_attributes>
<additional_attributes/> <additional_attributes/>
</element> </element>

@ -50,7 +50,7 @@ group=1</panel_attributes>
<w>130</w> <w>130</w>
<h>60</h> <h>60</h>
</coordinates> </coordinates>
<panel_attributes>StateFile <panel_attributes>Record
bg=yellow bg=yellow
halign=left halign=left
group=1</panel_attributes> group=1</panel_attributes>
@ -105,7 +105,7 @@ group=2</panel_attributes>
<w>130</w> <w>130</w>
<h>30</h> <h>30</h>
</coordinates> </coordinates>
<panel_attributes>StateFile <panel_attributes>Record
bg=yellow bg=yellow
halign=left halign=left
group=2</panel_attributes> group=2</panel_attributes>
@ -234,7 +234,7 @@ group=3</panel_attributes>
<w>130</w> <w>130</w>
<h>30</h> <h>30</h>
</coordinates> </coordinates>
<panel_attributes>StateFile <panel_attributes>Record
bg=yellow bg=yellow
halign=left halign=left
group=3</panel_attributes> group=3</panel_attributes>

@ -49,6 +49,11 @@
<artifactId>kotlin-stdlib-jdk8</artifactId> <artifactId>kotlin-stdlib-jdk8</artifactId>
<version>${kotlin.version}</version> <version>${kotlin.version}</version>
</dependency> </dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
<version>2.11.0</version>
</dependency>
</dependencies> </dependencies>
<build> <build>

@ -1,7 +1,7 @@
package com.olexyn.ensync; package com.olexyn.ensync;
import com.olexyn.ensync.artifacts.DataRoot; import com.olexyn.ensync.artifacts.DataRoot;
import com.olexyn.ensync.artifacts.StateFile; import com.olexyn.ensync.artifacts.Record;
import com.olexyn.ensync.artifacts.SyncDirectory; import com.olexyn.ensync.artifacts.SyncDirectory;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
@ -12,10 +12,11 @@ public class Flow implements Runnable {
private static final Logger LOGGER = LogUtil.get(Flow.class); private static final Logger LOGGER = LogUtil.get(Flow.class);
public static final long POLLING_PAUSE = 400; public static final long POLLING_PAUSE = 100;
private final AtomicBoolean running = new AtomicBoolean(false); private final AtomicBoolean running = new AtomicBoolean(false);
public void start() { public void start() {
LOGGER.info("START Flow.");
Thread worker = new Thread(this); Thread worker = new Thread(this);
worker.start(); worker.start();
} }
@ -28,19 +29,15 @@ public class Flow implements Runnable {
public void run() { public void run() {
running.set(true); running.set(true);
while (running.get()) { while (running.get()) {
synchronized(DataRoot.getSyncBundles()) { synchronized(DataRoot.getSyncBundles()) {
writeRecordIfMissing();
readOrMakeStateFile();
DataRoot.getSyncBundles().forEach( DataRoot.getSyncBundles().forEach(
syncBundle -> { syncBundle -> {
var syncDirectories = syncBundle.getSyncDirectories(); var syncDirectories = syncBundle.getSyncDirectories();
syncDirectories.forEach(this::doSyncDirectory); syncDirectories.forEach(this::sync);
} }
); );
} }
try { try {
LOGGER.info("Pausing... for " + POLLING_PAUSE + "ms."); LOGGER.info("Pausing... for " + POLLING_PAUSE + "ms.");
Thread.sleep(POLLING_PAUSE); Thread.sleep(POLLING_PAUSE);
@ -53,35 +50,36 @@ public class Flow implements Runnable {
/** /**
* *
*/ */
private void doSyncDirectory(SyncDirectory sd) { private void sync(SyncDirectory sDir) {
LOGGER.info("DO SYNC DIRECTORY"); LOGGER.info("DO SYNC " + sDir.directoryPath);
sd.readFileSystem(); var listFileSystem = sDir.readFileSystem();
LOGGER.info("# of files on FS: " + listFileSystem.size());
var record = sDir.readRecord();
sd.fillListOfLocallyCreatedFiles(); LOGGER.info("# of files on Record: " + record.size());
sd.makeListOfLocallyDeletedFiles(); var listCreated = sDir.fillListOfLocallyCreatedFiles(listFileSystem, record);
sd.makeListOfLocallyModifiedFiles(); LOGGER.info("# of files in Created: " + listCreated.size());
var listDeleted = sDir.makeListOfLocallyDeletedFiles(listFileSystem, record);
sd.doCreateOpsOnOtherSDs(); LOGGER.info("# of files in Deleted: " + listDeleted.size());
sd.doDeleteOpsOnOtherSDs(); var listModified = sDir.makeListOfLocallyModifiedFiles(listFileSystem);
sd.doModifyOpsOnOtherSDs(); LOGGER.info("# of files in Modified: " + listModified.size());
sDir.doCreateOpsOnOtherSDs(listCreated);
sd.writeStateFile(new StateFile(sd.directoryPath)); sDir.doDeleteOpsOnOtherSDs(listDeleted);
sDir.doModifyOpsOnOtherSDs(listModified);
sDir.writeRecord(new Record(sDir.directoryPath));
} }
/** /**
* For every single SyncDirectory try to read it's StateFile. <p> * For every single SyncDirectory try to read it's Record. <p>
* If the StateFile is missing, then create a StateFile. * If the Record is missing, then create a Record.
*/ */
private void readOrMakeStateFile() { private void writeRecordIfMissing() {
DataRoot.get().values().forEach(syncBundle -> { DataRoot.get().values().forEach(syncBundle -> {
for (var sd : syncBundle.syncDirectories.values()) { for (var sDir : syncBundle.syncDirectories.values()) {
var stateFile = new StateFile(sd.directoryPath); var record = new Record(sDir.directoryPath);
if (stateFile.exists()) { if (!record.exists()) {
LOGGER.info("READ-STATE-FILE"); sDir.writeRecord(new Record(sDir.directoryPath));
} else {
sd.writeStateFile(new StateFile(sd.directoryPath));
} }
} }
}); });

@ -7,8 +7,10 @@ import java.nio.file.Files;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.HashSet;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Set;
public class Tools { public class Tools {
@ -72,17 +74,12 @@ public class Tools {
} }
public Map<String, SyncFile> mapMinus(Map<String, SyncFile> fromA, Map<String, SyncFile> substractB) { public Set<String> setMinus(Set<String> fromA, Set<String> subtractB) {
Set<String> difference = new HashSet<>();
Map<String, SyncFile> difference = new HashMap<>(); for (var key : fromA) {
for (Map.Entry<String, SyncFile> entry : fromA.entrySet()) { if (fromA.contains(key) && !subtractB.contains(key)) {
String key = entry.getKey(); difference.add(key);
if (fromA.containsKey(key) && !substractB.containsKey(key)) {
SyncFile file = fromA.get(key);
difference.put(key, file);
} }
} }
return difference; return difference;
} }

@ -2,6 +2,8 @@ package com.olexyn.ensync.artifacts;
public interface Constants { public interface Constants {
String STATE_FILE_NAME = "state.ensync"; String STATE_FILE_NAME = "record.ensync";
String SPACE = " ";
String EMPTY = "";
} }

@ -3,7 +3,7 @@ package com.olexyn.ensync.artifacts
import java.io.File import java.io.File
import java.nio.file.Path import java.nio.file.Path
class StateFile(val targetPath: Path) { class Record(val targetPath: Path) {
fun getPath(): Path { fun getPath(): Path {
return targetPath.resolve(Constants.STATE_FILE_NAME) return targetPath.resolve(Constants.STATE_FILE_NAME)

@ -0,0 +1,17 @@
package com.olexyn.ensync.artifacts;
public class RecordFile extends SyncFile {
// Very IMPORTANT field. Allows to store lastModified as it is stored in the Record.
public long timeModifiedFromRecord = 0;
public RecordFile(SyncDirectory sDir, String absolutePath) {
super(sDir, absolutePath);
}
public void setTimeModifiedFromRecord(long value) {
timeModifiedFromRecord = value;
}
}

@ -1,40 +1,43 @@
package com.olexyn.ensync.artifacts; package com.olexyn.ensync.artifacts;
import com.olexyn.ensync.Execute;
import com.olexyn.ensync.LogUtil; import com.olexyn.ensync.LogUtil;
import com.olexyn.ensync.Tools; import com.olexyn.ensync.Tools;
import java.io.File; import java.io.File;
import java.io.FileInputStream;
import java.io.IOException; import java.io.IOException;
import java.math.BigInteger;
import java.nio.file.CopyOption;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Logger; import java.util.logging.Logger;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.Stream; import java.util.stream.Stream;
import static com.olexyn.ensync.artifacts.Constants.EMPTY;
import static com.olexyn.ensync.artifacts.Constants.SPACE;
/** /**
* A SyncDirectory is a singular occurrence of a directory in the filesystems. * A SyncDirectory is a singular occurrence of a directory in the filesystems.
*/ */
public class SyncDirectory { public class SyncDirectory {
private static final Logger LOGGER = LogUtil.get(SyncDirectory.class); private static final Logger LOGGER = LogUtil.get(SyncDirectory.class);
private final SyncBundle syncMap; private final SyncBundle syncMap;
public Path directoryPath; public Path directoryPath;
public Map<String, SyncFile> listCreated = new HashMap<>();
public final Map<String, SyncFile> listCreated = new HashMap<>(); public Map<String, SyncFile> listDeleted = new HashMap<>();
public final Map<String, SyncFile> listDeleted = new HashMap<>(); public Map<String, SyncFile> listModified = new HashMap<>();
public final Map<String, SyncFile> listModified = new HashMap<>();
Tools tools = new Tools(); Tools tools = new Tools();
Execute x = new Execute();
/** /**
* Create a SyncDirectory from realPath. * Create a SyncDirectory from realPath.
@ -42,10 +45,8 @@ public class SyncDirectory {
* @see SyncBundle * @see SyncBundle
*/ */
public SyncDirectory(Path directoryPath, SyncBundle syncMap) { public SyncDirectory(Path directoryPath, SyncBundle syncMap) {
this.directoryPath = directoryPath; this.directoryPath = directoryPath;
this.syncMap = syncMap; this.syncMap = syncMap;
} }
@ -53,7 +54,6 @@ public class SyncDirectory {
* Read the current state of the file system. * Read the current state of the file system.
*/ */
public Map<String, SyncFile> readFileSystem() { public Map<String, SyncFile> readFileSystem() {
//NOTE that the SyncFile().lastModifiedOld is not set here, so it is 0 by default.
return getFiles() return getFiles()
.map(file -> new SyncFile(this, file.getAbsolutePath())) .map(file -> new SyncFile(this, file.getAbsolutePath()))
.collect(Collectors.toMap( .collect(Collectors.toMap(
@ -62,24 +62,22 @@ public class SyncDirectory {
)); ));
} }
/** /**
* READ the contents of StateFile to Map. * READ the contents of Record to Map.
*/ */
public Map<String, SyncFile> readStateFile() { public Map<String, RecordFile> readRecord() {
Map<String, SyncFile> filemap = new HashMap<>(); Map<String, RecordFile> filemap = new HashMap<>();
var stateFile = new StateFile(directoryPath); var record = new Record(directoryPath);
List<String> lines = tools.fileToLines(stateFile.getPath().toFile()); List<String> lines = tools.fileToLines(record.getPath().toFile());
for (String line : lines) { for (String line : lines) {
// this is a predefined format: "modification-time path" // this is a predefined format: "<modification-time> <relative-path>"
String modTimeString = line.split(" ")[0]; var lineArr = line.split(SPACE);
long modTime = Long.parseLong(modTimeString); long modTime = Long.parseLong(lineArr[0]);
String sFilePath = lineArr[1];
RecordFile sfile = new RecordFile(this, sFilePath);
String sFilePath = line.replace(modTimeString + " ", ""); sfile.setTimeModifiedFromRecord(modTime);
SyncFile sfile = new SyncFile(this, sFilePath);
sfile.setTimeModifiedFromStateFile(modTime);
filemap.put(sFilePath, sfile); filemap.put(sFilePath, sfile);
} }
@ -91,58 +89,48 @@ public class SyncDirectory {
* Compare the OLD and NEW pools. * Compare the OLD and NEW pools.
* List is cleared and created each time. * List is cleared and created each time.
*/ */
public void fillListOfLocallyCreatedFiles() { public Map<String, SyncFile> fillListOfLocallyCreatedFiles(Map<String, SyncFile> listFileSystem, Map<String, RecordFile> record) {
listCreated.clear(); var listCreated = tools.setMinus(listFileSystem.keySet(), record.keySet());
var fromA = readFileSystem(); return listCreated.stream().collect(Collectors.toMap(key -> key, listFileSystem::get));
var substractB = readStateFile();
listCreated.putAll(tools.mapMinus(fromA, substractB));
} }
/** /**
* Compare the OLD and NEW pools. * Compare the OLD and NEW pools.
* List is cleared and created each time. * List is cleared and created each time.
*/ */
public void makeListOfLocallyDeletedFiles() { public Map<String, SyncFile> makeListOfLocallyDeletedFiles(Map<String, SyncFile> listFileSystem, Map<String, RecordFile> record) {
listDeleted.clear(); var listDeleted = tools.setMinus(record.keySet(), listFileSystem.keySet());
var fromA = readStateFile(); return listDeleted.stream().collect(Collectors.toMap(key -> key, record::get));
var substractB = readFileSystem();
var listDeleted = tools.mapMinus(fromA, substractB);
Map<String, SyncFile> swap = new HashMap<>();
for (var entry : listDeleted.entrySet()) {
String key = entry.getKey();
String parentKey = entry.getValue().getParent();
if (listDeleted.containsKey(parentKey) || swap.containsKey(parentKey)) {
swap.put(key, listDeleted.get(key));
}
}
listDeleted.putAll(tools.mapMinus(listDeleted, swap));
} }
/** /**
* Compare the OLD and NEW pools. * Compare the OLD and NEW pools.
* List is cleared and created each time. * List is cleared and created each time.
*/ */
public void makeListOfLocallyModifiedFiles() { public Map<String, SyncFile> makeListOfLocallyModifiedFiles(Map<String, SyncFile> listFileSystem) {
listModified.clear();
return listFileSystem.entrySet().stream().filter(
Map<String, SyncFile> stateFileMap = readStateFile(); fileEntry -> {
String fileKey = fileEntry.getKey();
for (var freshFileEntry : readFileSystem().entrySet()) { SyncFile file = fileEntry.getValue();
if (file.isDirectory()) { return false; } // no need to modify Directories, the Filesystem will do that, if a File changed.
String freshFileKey = freshFileEntry.getKey(); boolean isKnown = readRecord().containsKey(fileKey); // If KEY exists in OLD , thus FILE was NOT created.
SyncFile freshFile = freshFileEntry.getValue(); boolean isModified = file.lastModified() > file.lastModifiedFromRecord();
return isKnown && isModified;
if (freshFile.isDirectory()) { continue;} // no need to modify Directories, the Filesystem will do that, if a File changed. }
).collect(Collectors.toMap(Entry::getKey, Entry::getValue));
// If KEY exists in OLD , thus FILE was NOT created.
boolean oldFileExists = stateFileMap.containsKey(freshFileKey);
boolean fileIsFresher = freshFile.getTimeModified() > freshFile.getTimeModifiedFromStateFile();
if (oldFileExists && fileIsFresher) {
listModified.put(freshFileKey, freshFile);
} }
private String getMd5(Path path) {
try (var fos = new FileInputStream(path.toFile())) {
var m = MessageDigest.getInstance("MD5");
byte[] data = fos.readAllBytes();
m.update(data, 0, data.length);
var i = new BigInteger(1, m.digest());
return String.format("%1$032X", i);
} catch (NoSuchAlgorithmException | IOException e) {
LOGGER.info("File not found.");
return null;
} }
} }
@ -150,15 +138,21 @@ public class SyncDirectory {
* QUERY state of the filesystem at realPath. * QUERY state of the filesystem at realPath.
* WRITE the state of the filesystem to file. * WRITE the state of the filesystem to file.
*/ */
public void writeStateFile(StateFile stateFile) { public void writeRecord(Record record) {
List<String> outputList = new ArrayList<>(); List<String> outputList = new ArrayList<>();
getFiles().forEach( getFiles().forEach(
file -> { file -> {
String relativePath = file.getAbsolutePath() String relativePath = file.getAbsolutePath()
.replace(stateFile.getTargetPath().toString(), ""); .replace(record.getTargetPath().toString(), EMPTY);
outputList.add("" + file.lastModified() + " " + relativePath); var line = String.join(
SPACE,
String.valueOf(file.lastModified()),
relativePath
);
outputList.add(line);
}); });
tools.writeStringListToFile(stateFile.getPath().toString(), outputList); LOGGER.info("Writing " + outputList.size() + " files to " + record.getPath());
tools.writeStringListToFile(record.getPath().toString(), outputList);
} }
private Stream<File> getFiles() { private Stream<File> getFiles() {
@ -168,12 +162,12 @@ public class SyncDirectory {
.map(Path::toFile) .map(Path::toFile)
.filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME)); .filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME));
} catch (IOException e) { } catch (IOException e) {
LOGGER.severe("Could walk the file tree : StateFile will be empty."); LOGGER.severe("Could walk the file tree : Record will be empty.");
return Stream.empty(); return Stream.empty();
} }
} }
public void doCreateOpsOnOtherSDs() { public void doCreateOpsOnOtherSDs(Map<String, SyncFile> listCreated) {
for (var createdFile : listCreated.values()) { for (var createdFile : listCreated.values()) {
for (var otherFile : otherFiles(createdFile)) { for (var otherFile : otherFiles(createdFile)) {
writeFileIfNewer(createdFile, otherFile); writeFileIfNewer(createdFile, otherFile);
@ -181,7 +175,7 @@ public class SyncDirectory {
} }
} }
public void doDeleteOpsOnOtherSDs() { public void doDeleteOpsOnOtherSDs(Map<String, SyncFile> listDeleted) {
for (var deletedFile : listDeleted.values()) { for (var deletedFile : listDeleted.values()) {
for (var otherFile : otherFiles(deletedFile)) { for (var otherFile : otherFiles(deletedFile)) {
deleteFileIfNewer(deletedFile, otherFile); deleteFileIfNewer(deletedFile, otherFile);
@ -189,7 +183,7 @@ public class SyncDirectory {
} }
} }
public void doModifyOpsOnOtherSDs() { public void doModifyOpsOnOtherSDs(Map<String, SyncFile> listModified) {
for (var modifiedFile : listModified.values()) { for (var modifiedFile : listModified.values()) {
for (var otherFile : otherFiles(modifiedFile)) { for (var otherFile : otherFiles(modifiedFile)) {
writeFileIfNewer(modifiedFile, otherFile); writeFileIfNewer(modifiedFile, otherFile);
@ -210,11 +204,12 @@ public class SyncDirectory {
private void deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) { private void deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.exists()) { return; } if (!otherFile.exists()) { return; }
// if the otherFile was created with ensync it will have the == TimeModified. // if the otherFile was created with ensync it will have the == TimeModified.
if (thisFile.getTimeModified() >= otherFile.getTimeModified()) { if (thisFile.lastModified() >= otherFile.lastModified()) {
try { try {
Files.delete(Path.of(otherFile.getPath())); Files.delete(otherFile.toPath());
LOGGER.info("Deleted: " + otherFile.toPath());
} catch (IOException e) { } catch (IOException e) {
LOGGER.severe("Could not delete file."); LOGGER.severe("Could not delete: " + otherFile.toPath());
} }
} }
} }
@ -223,17 +218,48 @@ public class SyncDirectory {
* Overwrite other file if this file is newer. * Overwrite other file if this file is newer.
*/ */
private void writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) { private void writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (otherFile.exists() && thisFile.isOlder(otherFile)) { return; } LOGGER.info("Write from: " + thisFile.toPath());
if (thisFile.isFile()) { LOGGER.info(" to: " + otherFile.toPath());
if (!thisFile.isFile()) { return; }
if (otherFile.exists()) {
var thisMd5 = getMd5(thisFile.toPath());
var otherMd5 = getMd5(otherFile.toPath());
if (thisMd5 == null || otherMd5 == null) { return; }
if (thisMd5.equals(otherMd5)) {
dropAge(thisFile, otherFile);
return;
} else if (thisFile.isOlder(otherFile)) {
LOGGER.info("Did not override due to target being newer.");
return;
}
}
copyFile(thisFile, otherFile);
}
private void dropAge(SyncFile thisFile, SyncFile otherFile) {
if (thisFile.isOlder(otherFile)) {
otherFile.setLastModified(thisFile.lastModified());
LOGGER.info("Dropped age of OTHER: " + otherFile.toPath() + " to: " + otherFile.lastModified());
} else {
thisFile.setLastModified(otherFile.lastModified());
LOGGER.info("Dropped age of THIS: " + thisFile.toPath() + " to: " + thisFile.lastModified());
}
}
private void copyFile(SyncFile thisFile, SyncFile otherFile) {
try { try {
Files.copy( Files.copy(
Path.of(thisFile.getPath()), Path.of(thisFile.getPath()),
Path.of(otherFile.getPath()) Path.of(otherFile.getPath()),
StandardCopyOption.REPLACE_EXISTING,
StandardCopyOption.COPY_ATTRIBUTES
); );
otherFile.setLastModified(thisFile.lastModified()); LOGGER.info("Copied from: " + thisFile.toPath());
LOGGER.info(" to: " + otherFile.toPath());
} catch (IOException e) { } catch (IOException e) {
LOGGER.severe("Could not copy file."); LOGGER.severe("Could not copy file from: " + thisFile.toPath());
} LOGGER.severe(" to: " + otherFile.toPath());
e.printStackTrace();
} }
} }

@ -5,61 +5,60 @@ import com.olexyn.ensync.LogUtil;
import java.io.File; import java.io.File;
import java.util.logging.Logger; import java.util.logging.Logger;
import static com.olexyn.ensync.artifacts.Constants.EMPTY;
public class SyncFile extends File { public class SyncFile extends File {
private static final Logger LOGGER = LogUtil.get(SyncFile.class); private static final Logger LOGGER = LogUtil.get(SyncFile.class);
// Very IMPORTANT field. Allows to store lastModified as it is stored in the StateFile.
public long timeModifiedFromStateFile = 0;
private final String relativePath;
private final SyncDirectory sd;
public SyncFile(SyncDirectory sd, String absolutePath) { private final String relativePath;
private final SyncDirectory sDir;
public SyncFile(SyncDirectory sDir, String absolutePath) {
super(absolutePath); super(absolutePath);
this.sd = sd; this.sDir = sDir;
relativePath = this.getPath().replace(sd.directoryPath.toString(), ""); relativePath = this.getPath().replace(sDir.directoryPath.toString(), EMPTY);
} }
public String getRelativePath() { public String getRelativePath() {
return relativePath; return relativePath;
} }
public void setTimeModifiedFromStateFile(long value) {
timeModifiedFromStateFile = value;
}
public long getTimeModifiedFromStateFile() {
SyncFile record = sd.readStateFile().get(getRelativePath()); public long lastModifiedFromRecord() {
RecordFile record = getFromRecord();
if (record == null) { if (record == null) {
LOGGER.severe("Did not find record for file in StateFile. Setting modifiedDate to MIN, thus 0"); LOGGER.severe("Did not find record for " + this.toPath() + " in Record.");
return 0; LOGGER.severe("Returning -1 (never existed).");
return -1;
} }
return record.timeModifiedFromStateFile; return record.timeModifiedFromRecord;
}
public RecordFile getFromRecord() {
return sDir.readRecord().get(getRelativePath());
} }
/** /**
* If File exists on Disk get the TimeModified from there. * If File exists on Disk get the TimeModified from there.
* Else try to read it from StateFile. * Else try to read it from Record.
* Else return 0 ( = oldest possible time - a value of 0 can be seen as equal to "never existed"). * Else return 0 ( = oldest possible time - a value of 0 can be seen as equal to "never existed").
* EXAMPLES: * EXAMPLES:
* If a File was deleted, then the time will be taken from statefile. * If a File was deleted, then the time will be taken from statefile.
* If a File never existed, it will have time = 0, and thus will always be overwritten. * If a File never existed, it will have time = 0, and thus will always be overwritten.
*/ */
public long getTimeModified(){ @Override
if (exists()) { public long lastModified(){
return lastModified(); if (exists()) { return super.lastModified(); }
} return lastModifiedFromRecord();
if (sd.readStateFile().get(getPath()) != null) {
return getTimeModifiedFromStateFile();
}
return 0;
} }
public boolean isNewer(SyncFile otherFile) { public boolean isNewer(SyncFile otherFile) {
return this.getTimeModified() >= otherFile.getTimeModified(); LOGGER.info("" + this.lastModified() + " >= " + otherFile.lastModified());
return this.lastModified() >= otherFile.lastModified();
} }
public boolean isOlder(SyncFile otherFile) { public boolean isOlder(SyncFile otherFile) {

@ -21,6 +21,8 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.logging.Logger; import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
/** /**
* Perform the 15 test cases in TestCases.xlsx. * Perform the 15 test cases in TestCases.xlsx.
@ -35,17 +37,12 @@ public class FifteenTests {
final private static Tools tools = new Tools(); final private static Tools tools = new Tools();
private final static long FILE_OPS_PAUSE = 800; private final static long M1000 = 600;
private final static long WAIT_BEFORE_ASSERT = 4000;
private final static Execute x = new Execute();
private static final Path TEMP_DIR = Path.of(System.getProperty("user.dir") + "/src/test/temp"); private static final Path TEMP_DIR = Path.of(System.getProperty("user.dir") + "/src/test/temp");
private static final String RESOURCES_DIR = System.getProperty("user.dir") + "/src/test/resources";
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME; DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
private static final Path A_DIR = TEMP_DIR.resolve("a"); private static final Path A_DIR = TEMP_DIR.resolve("AAA");
private static final Path B_DIR = TEMP_DIR.resolve("b"); private static final Path B_DIR = TEMP_DIR.resolve("BBB");
private final TestFile aFile = new TestFile(A_DIR + "/testfile.txt"); private final TestFile aFile = new TestFile(A_DIR + "/testfile.txt");
private final TestFile bFile = new TestFile(B_DIR + "/testfile.txt"); private final TestFile bFile = new TestFile(B_DIR + "/testfile.txt");
@ -55,20 +52,22 @@ public class FifteenTests {
try { try {
stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " CREATED"); stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " CREATED");
tools.writeStringListToFile(file.getAbsolutePath(), stringList); tools.writeStringListToFile(file.getAbsolutePath(), stringList);
Thread.sleep(FILE_OPS_PAUSE); LOGGER.info("TEST CREATE: " + file.toPath());
Thread.sleep(M1000);
} catch (InterruptedException e) { } catch (InterruptedException e) {
System.out.println(""); System.out.println("");
} }
return stringList; return stringList;
} }
private List<String> updateFile(File file) { private List<String> modifyFile(File file) {
List<String> stringList = new ArrayList<>(); List<String> stringList = new ArrayList<>();
try { try {
stringList.addAll(tools.fileToLines(file)); stringList.addAll(tools.fileToLines(file));
stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " UPDATED"); stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " MODIFIED");
tools.writeStringListToFile(file.getAbsolutePath(), stringList); tools.writeStringListToFile(file.getAbsolutePath(), stringList);
Thread.sleep(FILE_OPS_PAUSE); LOGGER.info("TEST MODIFY: " + file.toPath());
Thread.sleep(M1000);
} catch (InterruptedException e) { } catch (InterruptedException e) {
System.out.println(""); System.out.println("");
} }
@ -79,40 +78,20 @@ public class FifteenTests {
private static void deleteFile(File file) { private static void deleteFile(File file) {
try { try {
Files.delete(file.toPath()); Files.delete(file.toPath());
Thread.sleep(FILE_OPS_PAUSE); LOGGER.info("TEST DELETE: " + file.toPath());
Thread.sleep(M1000);
} catch (IOException | InterruptedException e) { } catch (IOException | InterruptedException e) {
LOGGER.severe("Could not delete file."); LOGGER.severe("Could not delete file." + file.toPath());
}
}
private static void deleteRec(Path path) {
var file = path.toFile();
if (!file.exists()) {
return;
}
if (file.isDirectory()) {
try {
Files.walk(path)
.filter(subPath -> !subPath.equals(path))
.forEach(FifteenTests::deleteRec);
} catch (IOException e) {
LOGGER.severe("Could not walk path.");
}
}
try {
Files.delete(path);
} catch (IOException e) {
LOGGER.severe("Could not delete file.");
} }
} }
private void cleanDirs(Path... dirs) { private void cleanDirs(Path... dirs) {
for (var dir : dirs) { for (var dir : dirs) {
deleteRec(dir);
try { try {
FileUtils.deleteDirectory(dir.toFile());
Files.createDirectory(dir); Files.createDirectory(dir);
} catch (IOException e) { } catch (IOException e) {
LOGGER.severe("Could not clear dirs."); LOGGER.severe("Could not clear dirs. " + dir + e.getMessage());
} }
} }
} }
@ -120,9 +99,9 @@ public class FifteenTests {
SyncBundle syncBundle; SyncBundle syncBundle;
public static void waitBeforeAssert() { public static void waitX(long time) {
try { try {
Thread.sleep(WAIT_BEFORE_ASSERT); Thread.sleep(time);
} catch (InterruptedException ignored) {} } catch (InterruptedException ignored) {}
} }
@ -133,123 +112,157 @@ public class FifteenTests {
syncBundle.addDirectory(B_DIR); syncBundle.addDirectory(B_DIR);
cleanDirs(A_DIR, B_DIR); cleanDirs(A_DIR, B_DIR);
DataRoot.get().put(syncBundle.name, syncBundle); DataRoot.get().put(syncBundle.name, syncBundle);
FLOW.start();
} }
@After @After
public void reset() { public void reset() {
FLOW.stop(); FLOW.stop();
waitX(M1000);
cleanDirs(A_DIR, B_DIR); cleanDirs(A_DIR, B_DIR);
} }
@Test @Test
public void one() { public void test1() {
createFile(aFile); createFile(aFile);
waitX(M1000);
FLOW.start();
waitX(M1000);
deleteFile(aFile); deleteFile(aFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertFalse(aFile.exists()); Assert.assertFalse(aFile.exists());
Assert.assertFalse(bFile.exists()); Assert.assertFalse(bFile.exists());
} }
@Test @Test
public void three() { public void test2() {
createFile(aFile); createFile(aFile);
createFile(bFile); createFile(bFile);
waitX(M1000);
FLOW.start();
waitX(M1000);
deleteFile(aFile); deleteFile(aFile);
deleteFile(bFile); deleteFile(bFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertFalse(aFile.exists()); Assert.assertFalse(aFile.exists());
Assert.assertFalse(bFile.exists()); Assert.assertFalse(bFile.exists());
} }
@Test @Test
public void four() { public void test3() {
createFile(aFile); createFile(aFile);
waitX(M1000);
FLOW.start();
waitX(M1000);
deleteFile(aFile); deleteFile(aFile);
var bContent = createFile(bFile); var bContent = createFile(bFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertEquals(bContent, aFile.readContent()); Assert.assertEquals(bContent, aFile.readContent());
} }
@Test @Test
public void five() { public void test4() {
createFile(aFile); createFile(aFile);
createFile(bFile); createFile(bFile);
waitX(M1000);
FLOW.start();
waitX(M1000);
deleteFile(aFile); deleteFile(aFile);
var bContent = updateFile(bFile); var bContent = modifyFile(bFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertEquals(bContent, aFile.readContent()); Assert.assertEquals(bContent, aFile.readContent());
} }
@Test @Test
public void six() { public void test5() {
FLOW.start();
waitX(M1000);
var aContent = createFile(aFile); var aContent = createFile(aFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertEquals(aContent, bFile.readContent()); Assert.assertEquals(aContent, bFile.readContent());
} }
@Test @Test
public void eight() { public void test6() {
createFile(aFile);
createFile(bFile); createFile(bFile);
waitX(M1000);
FLOW.start();
waitX(M1000);
createFile(aFile);
deleteFile(bFile); deleteFile(bFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertFalse(aFile.exists()); Assert.assertFalse(aFile.exists());
Assert.assertFalse(bFile.exists()); Assert.assertFalse(bFile.exists());
} }
@Test @Test
public void nine() { public void test7() {
FLOW.start();
waitX(M1000);
createFile(aFile); createFile(aFile);
var bContent = createFile(bFile); var bContent = createFile(bFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertEquals(bContent, aFile.readContent()); Assert.assertEquals(bContent, aFile.readContent());
} }
@Test @Test
public void ten() { public void test8() {
createFile(bFile); createFile(bFile);
waitX(M1000);
FLOW.start();
waitX(M1000);
createFile(aFile); createFile(aFile);
var bContent = updateFile(bFile); var bContent = modifyFile(bFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertEquals(bContent, aFile.readContent()); Assert.assertEquals(bContent, aFile.readContent());
} }
@Test @Test
public void eleven() { public void test9() {
createFile(aFile); createFile(aFile);
var aContent = updateFile(aFile); waitX(M1000);
waitBeforeAssert(); FLOW.start();
waitX(M1000);
var aContent = modifyFile(aFile);
waitX(M1000);
Assert.assertEquals(aContent, bFile.readContent()); Assert.assertEquals(aContent, bFile.readContent());
} }
@Test @Test
public void thirteen() { public void test10() {
createFile(aFile); createFile(aFile);
createFile(bFile); createFile(bFile);
updateFile(aFile); waitX(M1000);
FLOW.start();
waitX(M1000);
modifyFile(aFile);
deleteFile(bFile); deleteFile(bFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertFalse(aFile.exists()); Assert.assertFalse(aFile.exists());
Assert.assertFalse(bFile.exists()); Assert.assertFalse(bFile.exists());
} }
@Test @Test
public void fourteen() { public void test11() {
createFile(aFile); createFile(aFile);
updateFile(aFile); waitX(M1000);
FLOW.start();
waitX(M1000);
modifyFile(aFile);
var bContent = createFile(bFile); var bContent = createFile(bFile);
waitBeforeAssert(); waitX(M1000);
Assert.assertEquals(bContent, aFile.readContent()); Assert.assertEquals(bContent, aFile.readContent());
} }
@Test @Test
public void fifteen() { public void test12() {
createFile(aFile); createFile(aFile);
createFile(bFile); createFile(bFile);
updateFile(aFile); waitX(M1000);
var bContent = updateFile(bFile); FLOW.start();
waitBeforeAssert(); waitX(M1000);
modifyFile(aFile);
var bContent = modifyFile(bFile);
waitX(M1000);
Assert.assertEquals(bContent, aFile.readContent()); Assert.assertEquals(bContent, aFile.readContent());
} }

@ -327,7 +327,7 @@ nothing</panel_attributes>
<h>40</h> <h>40</h>
</coordinates> </coordinates>
<panel_attributes> cp if newer <panel_attributes> cp if newer
try: time deletet = last time present in StateFile, else time deleted = 0 (~never existed) try: time deletet = last time present in Record, else time deleted = 0 (~never existed)
halign=left</panel_attributes> halign=left</panel_attributes>
<additional_attributes/> <additional_attributes/>
</element> </element>
@ -677,7 +677,7 @@ bg=red</panel_attributes>
<w>720</w> <w>720</w>
<h>30</h> <h>30</h>
</coordinates> </coordinates>
<panel_attributes>Deleted Files are tracked by their last existance in a StateFile.</panel_attributes> <panel_attributes>Deleted Files are tracked by their last existance in a Record.</panel_attributes>
<additional_attributes/> <additional_attributes/>
</element> </element>
<element> <element>

Loading…
Cancel
Save