public
/
ensync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

_ refactoring pt.2

Browse Source
pull/1/head
io42630 3 years ago
parent 4e6ad69877
commit 661d7d0c35
8 changed files with 140 additions and 229 deletions
Show Stats Download Patch File Download Diff File

1
.gitignore vendored
Unescape View File

@ -4,3 +4,4 @@
/ensync.iml /ensync.iml
*.xlsx# *.xlsx#
testfile.txt testfile.txt
*.log*

62
src/main/java/com/olexyn/ensync/Flow.java
Unescape View File

@ -3,65 +3,62 @@ package com.olexyn.ensync;
import com.olexyn.ensync.artifacts.DataRoot; import com.olexyn.ensync.artifacts.DataRoot;
import com.olexyn.ensync.artifacts.StateFile; import com.olexyn.ensync.artifacts.StateFile;
import com.olexyn.ensync.artifacts.SyncDirectory; import com.olexyn.ensync.artifacts.SyncDirectory;
import com.olexyn.ensync.artifacts.SyncBundle;
import java.io.File;
import java.util.Map.Entry;
import java.util.logging.Logger; import java.util.logging.Logger;
public class Flow implements Runnable { public class Flow implements Runnable {
private static final Logger LOGGER = LogUtil.get(Flow.class); private static final Logger LOGGER = LogUtil.get(Flow.class);
public static final long POLLING_PAUSE = 400;
Tools tools = new Tools(); /**
*
public long pollingPause = 200; */
@Override
public void run() { public void run() {
while (true) { while (true) {
synchronized(DataRoot.get()) { synchronized(DataRoot.getSyncBundles()) {
readOrMakeStateFile(); readOrMakeStateFile();
for (Entry<String, SyncBundle> syncMapEntry : DataRoot.get().entrySet()) { DataRoot.getSyncBundles().forEach(
syncBundle -> {
for (Entry<String, SyncDirectory> SDEntry : syncMapEntry.getValue().syncDirectories.entrySet()) { var syncDirectories = syncBundle.getSyncDirectories();
syncDirectories.forEach(this::doSyncDirectory);
doSyncDirectory(SDEntry.getValue());
}
} }
);
} }
try { try {
System.out.println("Pausing... for " + pollingPause + "ms."); System.out.println("Pausing... for " + POLLING_PAUSE + "ms.");
Thread.sleep(pollingPause); Thread.sleep(POLLING_PAUSE);
} catch (InterruptedException ignored) { } } catch (InterruptedException ignored) { }
} }
} }
/**
private void doSyncDirectory(SyncDirectory SD) { *
*/
private void doSyncDirectory(SyncDirectory sd) {
LOGGER.info("READ"); LOGGER.info("READ");
SD.readStateFromFS(); sd.readStateFromFS();
SD.listCreated = SD.makeListOfLocallyCreatedFiles();
SD.listDeleted = SD.makeListOfLocallyDeletedFiles();
SD.listModified = SD.makeListOfLocallyModifiedFiles();
SD.doCreateOpsOnOtherSDs(); sd.makeListOfLocallyCreatedFiles();
SD.doDeleteOpsOnOtherSDs(); sd.makeListOfLocallyDeletedFiles();
SD.doModifyOpsOnOtherSDs(); sd.makeListOfLocallyModifiedFiles();
SD.writeStateFile(new StateFile(SD.path)); sd.doCreateOpsOnOtherSDs();
} sd.doDeleteOpsOnOtherSDs();
sd.doModifyOpsOnOtherSDs();
sd.writeStateFile(new StateFile(sd.directoryPath));
}
/** /**
* For every single SyncDirectory try to read it's StateFile. <p> * For every single SyncDirectory try to read it's StateFile. <p>
@ -70,13 +67,14 @@ public class Flow implements Runnable {
private void readOrMakeStateFile() { private void readOrMakeStateFile() {
DataRoot.get().values().forEach(syncBundle -> { DataRoot.get().values().forEach(syncBundle -> {
for (var sd : syncBundle.syncDirectories.values()) { for (var sd : syncBundle.syncDirectories.values()) {
var stateFile = new StateFile(sd.path); var stateFile = new StateFile(sd.directoryPath);
if (stateFile.exists()) { if (stateFile.exists()) {
LOGGER.info("READ-STATE-FILE-" + sd.readStateFile()); LOGGER.info("READ-STATE-FILE-" + sd.readStateFile());
} else { } else {
sd.writeStateFile(new StateFile(sd.path)); sd.writeStateFile(new StateFile(sd.directoryPath));
} }
} }
}); });
} }
} }

2
src/main/java/com/olexyn/ensync/LogUtil.java
Unescape View File

@ -23,7 +23,7 @@ public class LogUtil {
System.setProperty("java.util.logging.SimpleFormatter.format", format); System.setProperty("java.util.logging.SimpleFormatter.format", format);
Logger logger = Logger.getLogger(c.getName()); Logger logger = Logger.getLogger(c.getName());
try { try {
String dir = System.getProperty("user.dir") + "/log/main.log"; String dir = System.getProperty("user.dir") + "/logs/main.log";
FileHandler fh = new FileHandler(dir, true); FileHandler fh = new FileHandler(dir, true);
fh.setFormatter(new SimpleFormatter() { fh.setFormatter(new SimpleFormatter() {
@Override @Override

6
src/main/java/com/olexyn/ensync/artifacts/DataRoot.java
Unescape View File

@ -1,5 +1,6 @@
package com.olexyn.ensync.artifacts; package com.olexyn.ensync.artifacts;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
public class DataRoot { public class DataRoot {
@ -16,4 +17,9 @@ public class DataRoot {
return mapOfSyncMaps; return mapOfSyncMaps;
} }
public static Collection<SyncBundle> getSyncBundles() {
return get().values();
}
} }

5
src/main/java/com/olexyn/ensync/artifacts/SyncBundle.java
Unescape View File

@ -4,6 +4,7 @@ package com.olexyn.ensync.artifacts;
import com.olexyn.ensync.Tools; import com.olexyn.ensync.Tools;
import java.io.File; import java.io.File;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
@ -25,6 +26,10 @@ public class SyncBundle {
this.name = name; this.name = name;
} }
public Collection<SyncDirectory> getSyncDirectories() {
return syncDirectories.values();
}
/** /**
* Creates a new SyncDirectory. <p> * Creates a new SyncDirectory. <p>
* Adds the created SyncDirectory to this SyncBundle. * Adds the created SyncDirectory to this SyncBundle.

253
src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java
Unescape View File

@ -1,7 +1,6 @@
package com.olexyn.ensync.artifacts; package com.olexyn.ensync.artifacts;
import com.olexyn.ensync.Execute; import com.olexyn.ensync.Execute;
import com.olexyn.ensync.Flow;
import com.olexyn.ensync.LogUtil; import com.olexyn.ensync.LogUtil;
import com.olexyn.ensync.Tools; import com.olexyn.ensync.Tools;
@ -11,10 +10,13 @@ import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.logging.Logger; import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/** /**
* A SyncDirectory is a singular occurrence of a directory in the filesystems. * A SyncDirectory is a singular occurrence of a directory in the filesystems.
@ -23,16 +25,12 @@ public class SyncDirectory {
private static final Logger LOGGER = LogUtil.get(SyncDirectory.class); private static final Logger LOGGER = LogUtil.get(SyncDirectory.class);
private String flowState;
private SyncDirectory thisSD = this;
private final SyncBundle syncMap; private final SyncBundle syncMap;
public String path = null; public String directoryPath;
public Map<String, SyncFile> listCreated = new HashMap<>(); public final Map<String, SyncFile> listCreated = new HashMap<>();
public Map<String, SyncFile> listDeleted = new HashMap<>(); public final Map<String, SyncFile> listDeleted = new HashMap<>();
public Map<String, SyncFile> listModified = new HashMap<>(); public final Map<String, SyncFile> listModified = new HashMap<>();
Tools tools = new Tools(); Tools tools = new Tools();
@ -43,9 +41,9 @@ public class SyncDirectory {
* *
* @see SyncBundle * @see SyncBundle
*/ */
public SyncDirectory(String path, SyncBundle syncMap) { public SyncDirectory(String directoryPath, SyncBundle syncMap) {
this.path = path; this.directoryPath = directoryPath;
this.syncMap = syncMap; this.syncMap = syncMap;
} }
@ -55,21 +53,12 @@ public class SyncDirectory {
* Get the current state by using the `find` command. * Get the current state by using the `find` command.
*/ */
public Map<String, SyncFile> readStateFromFS() { public Map<String, SyncFile> readStateFromFS() {
//NOTE that the SFile().lastModifiedOld is not set here, so it is 0 by default. //NOTE that the SyncFile().lastModifiedOld is not set here, so it is 0 by default.
Map<String, SyncFile> filemap = new HashMap<>(); Map<String, SyncFile> filemap = new HashMap<>();
getFiles()
Execute.TwoBr find = x.execute(new String[]{ .map(file -> new SyncFile(this, file.getAbsolutePath()))
"find", .forEach(file -> filemap.put(file.getAbsolutePath(), file)
path );
});
List<String> pathList = tools.brToListString(find.output);
for (String filePath : pathList) {
SyncFile file = new SyncFile(this, filePath);
filemap.put(filePath, file);
}
return filemap; return filemap;
} }
@ -79,7 +68,7 @@ public class SyncDirectory {
*/ */
public Map<String, SyncFile> readStateFile() { public Map<String, SyncFile> readStateFile() {
Map<String, SyncFile> filemap = new HashMap<>(); Map<String, SyncFile> filemap = new HashMap<>();
var stateFile = new StateFile(path); var stateFile = new StateFile(directoryPath);
List<String> lines = tools.fileToLines(new File(stateFile.getPath())); List<String> lines = tools.fileToLines(new File(stateFile.getPath()));
for (String line : lines) { for (String line : lines) {
@ -117,10 +106,10 @@ public class SyncDirectory {
*/ */
public Map<String, SyncFile> makeListOfLocallyDeletedFiles() { public Map<String, SyncFile> makeListOfLocallyDeletedFiles() {
Map<String, SyncFile> fromA = readStateFile(); var fromA = readStateFile();
Map<String, SyncFile> substractB = readStateFromFS(); var substractB = readStateFromFS();
Map<String, SyncFile> listDeleted = tools.mapMinus(fromA, substractB); var listDeleted = tools.mapMinus(fromA, substractB);
Map<String, SyncFile> swap = new HashMap<>(); Map<String, SyncFile> swap = new HashMap<>();
@ -143,7 +132,8 @@ public class SyncDirectory {
* Compare the OLD and NEW pools. * Compare the OLD and NEW pools.
* List is cleared and created each time. * List is cleared and created each time.
*/ */
public Map<String, SyncFile> makeListOfLocallyModifiedFiles() { public void makeListOfLocallyModifiedFiles() {
Map<String, SyncFile> listModified = new HashMap<>(); Map<String, SyncFile> listModified = new HashMap<>();
@ -164,7 +154,6 @@ public class SyncDirectory {
listModified.put(freshFileKey, freshFile); listModified.put(freshFileKey, freshFile);
} }
} }
return listModified;
} }
@ -174,198 +163,88 @@ public class SyncDirectory {
*/ */
public void writeStateFile(StateFile stateFile) { public void writeStateFile(StateFile stateFile) {
List<String> outputList = new ArrayList<>(); List<String> outputList = new ArrayList<>();
getFiles().forEach(
try { file -> {
Files.walk(Paths.get(path))
.filter(Files::isRegularFile)
.map(Path::toFile)
.filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME))
.forEach(file -> {
String relativePath = file.getAbsolutePath() String relativePath = file.getAbsolutePath()
.replace(stateFile.getTargetPath(), ""); .replace(stateFile.getTargetPath(), "");
outputList.add("" + file.lastModified() + " " + relativePath); outputList.add("" + file.lastModified() + " " + relativePath);
}); });
} catch (IOException e) {
LOGGER.severe("Could walk the file tree : StateFile will be empty.");
}
tools.writeStringListToFile(stateFile.getPath(), outputList); tools.writeStringListToFile(stateFile.getPath(), outputList);
} }
private Stream<File> getFiles() {
private class Info { try {
return Files.walk(Paths.get(directoryPath))
private String thisFilePath; .filter(Files::isRegularFile)
private String otherFilePath; .map(Path::toFile)
private String otherParentPath; .filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME));
private File otherParentFile; } catch (IOException e) {
LOGGER.severe("Could walk the file tree : StateFile will be empty.");
return Stream.empty();
private Info(SyncDirectory thisSD, SyncFile sFile, SyncDirectory otherSD) {
// Example:
// syncDirectory /foo
// otherSyncDirectory /bar
// createdFile /foo/hello/created-file.gif
// relativePath /hello/created-file.gif
String relativePath = sFile.getPath().replace(thisSD.path, "");
this.thisFilePath = sFile.getPath();
this.otherFilePath = otherSD.path + relativePath;
File otherFile = new File(otherFilePath);
this.otherParentPath = otherFile.getParent();
this.otherParentFile = new File(otherParentPath);
} }
} }
public void doCreateOpsOnOtherSDs() { public void doCreateOpsOnOtherSDs() {
for (var createdFile : listCreated.values()) {
for (var entry : listCreated.entrySet()) { for (var otherFile : otherFiles(createdFile)) {
SyncFile createdFile = entry.getValue(); writeFileIfNewer(createdFile, otherFile);
for (var otherEntry : syncMap.syncDirectories.entrySet()) {
SyncDirectory otherSD = otherEntry.getValue();
if (this.equals(otherSD)) { continue; }
Info info = new Info(this, createdFile, otherSD);
writeFileIfNewer(info, createdFile, otherSD);
} }
} }
} }
/**
*
*/
public void doDeleteOpsOnOtherSDs() { public void doDeleteOpsOnOtherSDs() {
for (var deletedFile : listDeleted.values()) {
for (var entry : listDeleted.entrySet()) { for (var otherFile : otherFiles(deletedFile)) {
SyncFile deletedFile = entry.getValue(); deleteFileIfNewer(deletedFile, otherFile);
for (var otherEntry : syncMap.syncDirectories.entrySet()) {
SyncDirectory otherSD = otherEntry.getValue();
if (this.equals(otherSD)) { continue; }
Info info = new Info(thisSD, deletedFile, otherSD);
deleteFile(info, thisSD, deletedFile, otherSD);
}
} }
} }
private void deleteFile(Info info, SyncDirectory thisSD, SyncFile thisFile, SyncDirectory otherSD) {
SyncFile otherFile = new SyncFile(otherSD, otherSD.path + thisFile.relativePath);
if (!otherFile.exists()) { return;}
// if the otherFile was created with ensync it will have the == TimeModified.
long thisFileTimeModified = thisFile.getTimeModified();
long otherFileTimeModified = otherFile.getTimeModified();
if (thisFile.getTimeModified() >= otherFile.getTimeModified()) {
List<String> cmd = List.of("rm", "-r", info.otherFilePath);
x.execute(cmd);
}
} }
public void doModifyOpsOnOtherSDs() { public void doModifyOpsOnOtherSDs() {
for (var modifiedFile : listModified.values()) {
for (var entry : listModified.entrySet()) { for (var otherFile : otherFiles(modifiedFile)) {
SyncFile modifiedFile = entry.getValue(); writeFileIfNewer(modifiedFile, otherFile);
for (var otherEntry : syncMap.syncDirectories.entrySet()) {
SyncDirectory otherSD = otherEntry.getValue();
if (this.equals(otherSD)) { continue; }
Info info = new Info(this, modifiedFile, otherSD);
writeFileIfNewer(info, modifiedFile, otherSD);
} }
} }
} }
/*** private Collection<SyncFile> otherFiles(SyncFile thisFile) {
* return syncMap.getSyncDirectories().stream()
* @param info .filter(sd -> !this.equals(sd))
* @param thisFile .map(thisFile::otherFile)
* @param otherSD .collect(Collectors.toList());
*/
private void writeFileIfNewer(Info info, SyncFile thisFile, SyncDirectory otherSD) {
SyncFile otherFile = new SyncFile(otherSD, otherSD.path + thisFile.relativePath);
if (otherFile.exists() && thisFile.getTimeModified() < otherFile.getTimeModified()) { return;}
if (thisFile.isDirectory() && !otherFile.exists()) {
List<String> cmd = List.of("mkdir", "-p", info.otherFilePath);
x.execute(cmd);
return;
} }
if (thisFile.isFile()) { /**
* Delete other file if this file is newer.
if (!info.otherParentFile.exists()) { */
makeParentChain(otherFile, thisFile); private void deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.exists()) { return; }
// if the otherFile was created with ensync it will have the == TimeModified.
if (thisFile.getTimeModified() >= otherFile.getTimeModified()) {
try {
Files.delete(Path.of(otherFile.getPath()));
} catch (IOException e) {
LOGGER.severe("Could not delete file.");
} }
List<String> cmd = List.of("cp", "-p", info.thisFilePath, info.otherFilePath);
x.execute(cmd);
copyModifDate(thisFile.getParentFile(), otherFile.getParentFile());
} }
} }
/** /**
* @param otherFile * Overwrite other file if this file is newer.
* @param thisFile
*/ */
private void makeParentChain(File otherFile, File thisFile) { private void writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (otherFile.exists() && thisFile.isOlder(otherFile)) { return; }
if (thisFile.isFile()) {
try { try {
File otherParent = new File(otherFile.getParent()); Files.copy(
File thisParent = new File(thisFile.getParent()); Path.of(thisFile.getPath()),
Path.of(otherFile.getPath())
if (!otherParent.exists()) { );
makeParentChain(otherParent, thisParent); } catch (IOException e) {
makeParentChain(otherFile, thisFile); LOGGER.severe("Could not copy file.");
} else if (thisFile.isDirectory()) {
List<String> cmd = List.of("mkdir", otherFile.getPath());
x.execute(cmd);
cmd = List.of("stat", "--format", "%y", thisFile.getPath());
String mDate = x.execute(cmd).output.readLine();
cmd = List.of("touch", "-m", "--date=" + mDate, otherFile.getPath());
String error = x.execute(cmd).error.readLine();
int br = 0;
} }
} catch (Exception ignored) {}
} }
private void copyModifDate(File fromFile, File toFile) {
try {
List<String> cmd = List.of("stat", "--format", "%y", fromFile.getPath());
String mDate = x.execute(cmd).output.readLine();
cmd = List.of("touch", "-m", "--date=" + mDate, toFile.getPath());
x.execute(cmd);
} catch (Exception ignored) {}
} }
} }

20
src/main/java/com/olexyn/ensync/artifacts/SyncFile.java
Unescape View File

@ -1,7 +1,6 @@
package com.olexyn.ensync.artifacts; package com.olexyn.ensync.artifacts;
import java.io.File; import java.io.File;
import java.util.Map;
public class SyncFile extends File { public class SyncFile extends File {
@ -16,7 +15,7 @@ public class SyncFile extends File {
super(pathname); super(pathname);
this.sd = sd; this.sd = sd;
relativePath = this.getPath().replace(sd.path, ""); relativePath = this.getPath().replace(sd.directoryPath, "");
} }
public void setTimeModifiedFromStateFile(long value) { public void setTimeModifiedFromStateFile(long value) {
@ -39,13 +38,26 @@ public class SyncFile extends File {
* If a File never existed, it will have time = 0, and thus will always be overwritten. * If a File never existed, it will have time = 0, and thus will always be overwritten.
*/ */
public long getTimeModified(){ public long getTimeModified(){
if (this.exists()){ if (exists()) {
return lastModified(); return lastModified();
} }
if (sd.readStateFile().get(this.getPath())!=null){ if (sd.readStateFile().get(getPath()) != null) {
return getTimeModifiedFromStateFile(); return getTimeModifiedFromStateFile();
} }
return 0; return 0;
} }
public boolean isNewer(SyncFile otherFile) {
return this.getTimeModified() >= otherFile.getTimeModified();
}
public boolean isOlder(SyncFile otherFile) {
return !isNewer(otherFile);
}
public SyncFile otherFile(SyncDirectory otherSd) {
return new SyncFile(otherSd, otherSd.directoryPath + this.relativePath);
}
} }

10
src/test/test_cases.csv
Unescape View File

@ -0,0 +1,10 @@
Case
1
2
3
4
5
6
7
8
9
1 Case
2 1
3 2
4 3
5 4
6 5
7 6
8 7
9 8
10 9
Write Preview
Loading…
Cancel
Save