public
/
ensync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

_ refactoring pt.2

Browse Source
pull/1/head
io42630 3 years ago
parent 4e6ad69877
commit 661d7d0c35
8 changed files with 140 additions and 229 deletions
Show Stats Download Patch File Download Diff File

1
.gitignore vendored
Unescape View File

@ -4,3 +4,4 @@
/ensync.iml
*.xlsx#
testfile.txt
*.log*

62
src/main/java/com/olexyn/ensync/Flow.java
Unescape View File

@ -3,65 +3,62 @@ package com.olexyn.ensync;
import com.olexyn.ensync.artifacts.DataRoot;
import com.olexyn.ensync.artifacts.StateFile;
import com.olexyn.ensync.artifacts.SyncDirectory;
import com.olexyn.ensync.artifacts.SyncBundle;
import java.io.File;
import java.util.Map.Entry;
import java.util.logging.Logger;
public class Flow implements Runnable {
private static final Logger LOGGER = LogUtil.get(Flow.class);
public static final long POLLING_PAUSE = 400;
Tools tools = new Tools();
public long pollingPause = 200;
/**
*
*/
@Override
public void run() {
while (true) {
synchronized(DataRoot.get()) {
synchronized(DataRoot.getSyncBundles()) {
readOrMakeStateFile();
for (Entry<String, SyncBundle> syncMapEntry : DataRoot.get().entrySet()) {
for (Entry<String, SyncDirectory> SDEntry : syncMapEntry.getValue().syncDirectories.entrySet()) {
doSyncDirectory(SDEntry.getValue());
DataRoot.getSyncBundles().forEach(
syncBundle -> {
var syncDirectories = syncBundle.getSyncDirectories();
syncDirectories.forEach(this::doSyncDirectory);
}
}
);
}
try {
System.out.println("Pausing... for " + pollingPause + "ms.");
Thread.sleep(pollingPause);
System.out.println("Pausing... for " + POLLING_PAUSE + "ms.");
Thread.sleep(POLLING_PAUSE);
} catch (InterruptedException ignored) { }
}
}
private void doSyncDirectory(SyncDirectory SD) {
/**
*
*/
private void doSyncDirectory(SyncDirectory sd) {
LOGGER.info("READ");
SD.readStateFromFS();
sd.readStateFromFS();
SD.listCreated = SD.makeListOfLocallyCreatedFiles();
SD.listDeleted = SD.makeListOfLocallyDeletedFiles();
SD.listModified = SD.makeListOfLocallyModifiedFiles();
SD.doCreateOpsOnOtherSDs();
SD.doDeleteOpsOnOtherSDs();
SD.doModifyOpsOnOtherSDs();
sd.makeListOfLocallyCreatedFiles();
sd.makeListOfLocallyDeletedFiles();
sd.makeListOfLocallyModifiedFiles();
SD.writeStateFile(new StateFile(SD.path));
}
sd.doCreateOpsOnOtherSDs();
sd.doDeleteOpsOnOtherSDs();
sd.doModifyOpsOnOtherSDs();
sd.writeStateFile(new StateFile(sd.directoryPath));
}
/**
* For every single SyncDirectory try to read it's StateFile. <p>
@ -70,13 +67,14 @@ public class Flow implements Runnable {
private void readOrMakeStateFile() {
DataRoot.get().values().forEach(syncBundle -> {
for (var sd : syncBundle.syncDirectories.values()) {
var stateFile = new StateFile(sd.path);
var stateFile = new StateFile(sd.directoryPath);
if (stateFile.exists()) {
LOGGER.info("READ-STATE-FILE-" + sd.readStateFile());
} else {
sd.writeStateFile(new StateFile(sd.path));
sd.writeStateFile(new StateFile(sd.directoryPath));
}
}
});
}
}

2
src/main/java/com/olexyn/ensync/LogUtil.java
Unescape View File

@ -23,7 +23,7 @@ public class LogUtil {
System.setProperty("java.util.logging.SimpleFormatter.format", format);
Logger logger = Logger.getLogger(c.getName());
try {
String dir = System.getProperty("user.dir") + "/log/main.log";
String dir = System.getProperty("user.dir") + "/logs/main.log";
FileHandler fh = new FileHandler(dir, true);
fh.setFormatter(new SimpleFormatter() {
@Override

6
src/main/java/com/olexyn/ensync/artifacts/DataRoot.java
Unescape View File

@ -1,5 +1,6 @@
package com.olexyn.ensync.artifacts;
import java.util.Collection;
import java.util.HashMap;
public class DataRoot {
@ -16,4 +17,9 @@ public class DataRoot {
return mapOfSyncMaps;
}
public static Collection<SyncBundle> getSyncBundles() {
return get().values();
}
}

5
src/main/java/com/olexyn/ensync/artifacts/SyncBundle.java
Unescape View File

@ -4,6 +4,7 @@ package com.olexyn.ensync.artifacts;
import com.olexyn.ensync.Tools;
import java.io.File;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
@ -25,6 +26,10 @@ public class SyncBundle {
this.name = name;
}
public Collection<SyncDirectory> getSyncDirectories() {
return syncDirectories.values();
}
/**
* Creates a new SyncDirectory. <p>
* Adds the created SyncDirectory to this SyncBundle.

261
src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java
Unescape View File

@ -1,7 +1,6 @@
package com.olexyn.ensync.artifacts;
import com.olexyn.ensync.Execute;
import com.olexyn.ensync.Flow;
import com.olexyn.ensync.LogUtil;
import com.olexyn.ensync.Tools;
@ -11,10 +10,13 @@ import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* A SyncDirectory is a singular occurrence of a directory in the filesystems.
@ -23,16 +25,12 @@ public class SyncDirectory {
private static final Logger LOGGER = LogUtil.get(SyncDirectory.class);
private String flowState;
private SyncDirectory thisSD = this;
private final SyncBundle syncMap;
public String path = null;
public String directoryPath;
public Map<String, SyncFile> listCreated = new HashMap<>();
public Map<String, SyncFile> listDeleted = new HashMap<>();
public Map<String, SyncFile> listModified = new HashMap<>();
public final Map<String, SyncFile> listCreated = new HashMap<>();
public final Map<String, SyncFile> listDeleted = new HashMap<>();
public final Map<String, SyncFile> listModified = new HashMap<>();
Tools tools = new Tools();
@ -43,9 +41,9 @@ public class SyncDirectory {
*
* @see SyncBundle
*/
public SyncDirectory(String path, SyncBundle syncMap) {
public SyncDirectory(String directoryPath, SyncBundle syncMap) {
this.path = path;
this.directoryPath = directoryPath;
this.syncMap = syncMap;
}
@ -55,21 +53,12 @@ public class SyncDirectory {
* Get the current state by using the `find` command.
*/
public Map<String, SyncFile> readStateFromFS() {
//NOTE that the SFile().lastModifiedOld is not set here, so it is 0 by default.
//NOTE that the SyncFile().lastModifiedOld is not set here, so it is 0 by default.
Map<String, SyncFile> filemap = new HashMap<>();
Execute.TwoBr find = x.execute(new String[]{
"find",
path
});
List<String> pathList = tools.brToListString(find.output);
for (String filePath : pathList) {
SyncFile file = new SyncFile(this, filePath);
filemap.put(filePath, file);
}
getFiles()
.map(file -> new SyncFile(this, file.getAbsolutePath()))
.forEach(file -> filemap.put(file.getAbsolutePath(), file)
);
return filemap;
}
@ -79,7 +68,7 @@ public class SyncDirectory {
*/
public Map<String, SyncFile> readStateFile() {
Map<String, SyncFile> filemap = new HashMap<>();
var stateFile = new StateFile(path);
var stateFile = new StateFile(directoryPath);
List<String> lines = tools.fileToLines(new File(stateFile.getPath()));
for (String line : lines) {
@ -117,10 +106,10 @@ public class SyncDirectory {
*/
public Map<String, SyncFile> makeListOfLocallyDeletedFiles() {
Map<String, SyncFile> fromA = readStateFile();
Map<String, SyncFile> substractB = readStateFromFS();
var fromA = readStateFile();
var substractB = readStateFromFS();
Map<String, SyncFile> listDeleted = tools.mapMinus(fromA, substractB);
var listDeleted = tools.mapMinus(fromA, substractB);
Map<String, SyncFile> swap = new HashMap<>();
@ -143,7 +132,8 @@ public class SyncDirectory {
* Compare the OLD and NEW pools.
* List is cleared and created each time.
*/
public Map<String, SyncFile> makeListOfLocallyModifiedFiles() {
public void makeListOfLocallyModifiedFiles() {
Map<String, SyncFile> listModified = new HashMap<>();
@ -164,7 +154,6 @@ public class SyncDirectory {
listModified.put(freshFileKey, freshFile);
}
}
return listModified;
}
@ -174,198 +163,88 @@ public class SyncDirectory {
*/
public void writeStateFile(StateFile stateFile) {
List<String> outputList = new ArrayList<>();
getFiles().forEach(
file -> {
String relativePath = file.getAbsolutePath()
.replace(stateFile.getTargetPath(), "");
outputList.add("" + file.lastModified() + " " + relativePath);
});
tools.writeStringListToFile(stateFile.getPath(), outputList);
}
private Stream<File> getFiles() {
try {
Files.walk(Paths.get(path))
return Files.walk(Paths.get(directoryPath))
.filter(Files::isRegularFile)
.map(Path::toFile)
.filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME))
.forEach(file -> {
String relativePath = file.getAbsolutePath()
.replace(stateFile.getTargetPath(), "");
outputList.add("" + file.lastModified() + " " + relativePath);
});
.filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME));
} catch (IOException e) {
LOGGER.severe("Could walk the file tree : StateFile will be empty.");
}
tools.writeStringListToFile(stateFile.getPath(), outputList);
}
private class Info {
private String thisFilePath;
private String otherFilePath;
private String otherParentPath;
private File otherParentFile;
private Info(SyncDirectory thisSD, SyncFile sFile, SyncDirectory otherSD) {
// Example:
// syncDirectory /foo
// otherSyncDirectory /bar
// createdFile /foo/hello/created-file.gif
// relativePath /hello/created-file.gif
String relativePath = sFile.getPath().replace(thisSD.path, "");
this.thisFilePath = sFile.getPath();
this.otherFilePath = otherSD.path + relativePath;
File otherFile = new File(otherFilePath);
this.otherParentPath = otherFile.getParent();
this.otherParentFile = new File(otherParentPath);
return Stream.empty();
}
}
public void doCreateOpsOnOtherSDs() {
for (var entry : listCreated.entrySet()) {
SyncFile createdFile = entry.getValue();
for (var otherEntry : syncMap.syncDirectories.entrySet()) {
SyncDirectory otherSD = otherEntry.getValue();
if (this.equals(otherSD)) { continue; }
Info info = new Info(this, createdFile, otherSD);
writeFileIfNewer(info, createdFile, otherSD);
for (var createdFile : listCreated.values()) {
for (var otherFile : otherFiles(createdFile)) {
writeFileIfNewer(createdFile, otherFile);
}
}
}
/**
*
*/
public void doDeleteOpsOnOtherSDs() {
for (var entry : listDeleted.entrySet()) {
SyncFile deletedFile = entry.getValue();
for (var otherEntry : syncMap.syncDirectories.entrySet()) {
SyncDirectory otherSD = otherEntry.getValue();
if (this.equals(otherSD)) { continue; }
Info info = new Info(thisSD, deletedFile, otherSD);
deleteFile(info, thisSD, deletedFile, otherSD);
for (var deletedFile : listDeleted.values()) {
for (var otherFile : otherFiles(deletedFile)) {
deleteFileIfNewer(deletedFile, otherFile);
}
}
}
private void deleteFile(Info info, SyncDirectory thisSD, SyncFile thisFile, SyncDirectory otherSD) {
SyncFile otherFile = new SyncFile(otherSD, otherSD.path + thisFile.relativePath);
if (!otherFile.exists()) { return;}
// if the otherFile was created with ensync it will have the == TimeModified.
long thisFileTimeModified = thisFile.getTimeModified();
long otherFileTimeModified = otherFile.getTimeModified();
if (thisFile.getTimeModified() >= otherFile.getTimeModified()) {
List<String> cmd = List.of("rm", "-r", info.otherFilePath);
x.execute(cmd);
}
}
public void doModifyOpsOnOtherSDs() {
for (var entry : listModified.entrySet()) {
SyncFile modifiedFile = entry.getValue();
for (var otherEntry : syncMap.syncDirectories.entrySet()) {
SyncDirectory otherSD = otherEntry.getValue();
if (this.equals(otherSD)) { continue; }
Info info = new Info(this, modifiedFile, otherSD);
writeFileIfNewer(info, modifiedFile, otherSD);
for (var modifiedFile : listModified.values()) {
for (var otherFile : otherFiles(modifiedFile)) {
writeFileIfNewer(modifiedFile, otherFile);
}
}
}
/***
*
* @param info
* @param thisFile
* @param otherSD
*/
private void writeFileIfNewer(Info info, SyncFile thisFile, SyncDirectory otherSD) {
SyncFile otherFile = new SyncFile(otherSD, otherSD.path + thisFile.relativePath);
if (otherFile.exists() && thisFile.getTimeModified() < otherFile.getTimeModified()) { return;}
if (thisFile.isDirectory() && !otherFile.exists()) {
List<String> cmd = List.of("mkdir", "-p", info.otherFilePath);
x.execute(cmd);
return;
}
if (thisFile.isFile()) {
private Collection<SyncFile> otherFiles(SyncFile thisFile) {
return syncMap.getSyncDirectories().stream()
.filter(sd -> !this.equals(sd))
.map(thisFile::otherFile)
.collect(Collectors.toList());
}
if (!info.otherParentFile.exists()) {
makeParentChain(otherFile, thisFile);
/**
* Delete other file if this file is newer.
*/
private void deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.exists()) { return; }
// if the otherFile was created with ensync it will have the == TimeModified.
if (thisFile.getTimeModified() >= otherFile.getTimeModified()) {
try {
Files.delete(Path.of(otherFile.getPath()));
} catch (IOException e) {
LOGGER.severe("Could not delete file.");
}
List<String> cmd = List.of("cp", "-p", info.thisFilePath, info.otherFilePath);
x.execute(cmd);
copyModifDate(thisFile.getParentFile(), otherFile.getParentFile());
}
}
/**
* @param otherFile
* @param thisFile
* Overwrite other file if this file is newer.
*/
private void makeParentChain(File otherFile, File thisFile) {
try {
File otherParent = new File(otherFile.getParent());
File thisParent = new File(thisFile.getParent());
if (!otherParent.exists()) {
makeParentChain(otherParent, thisParent);
makeParentChain(otherFile, thisFile);
} else if (thisFile.isDirectory()) {
List<String> cmd = List.of("mkdir", otherFile.getPath());
x.execute(cmd);
cmd = List.of("stat", "--format", "%y", thisFile.getPath());
String mDate = x.execute(cmd).output.readLine();
cmd = List.of("touch", "-m", "--date=" + mDate, otherFile.getPath());
String error = x.execute(cmd).error.readLine();
int br = 0;
private void writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (otherFile.exists() && thisFile.isOlder(otherFile)) { return; }
if (thisFile.isFile()) {
try {
Files.copy(
Path.of(thisFile.getPath()),
Path.of(otherFile.getPath())
);
} catch (IOException e) {
LOGGER.severe("Could not copy file.");
}
} catch (Exception ignored) {}
}
private void copyModifDate(File fromFile, File toFile) {
try {
List<String> cmd = List.of("stat", "--format", "%y", fromFile.getPath());
String mDate = x.execute(cmd).output.readLine();
cmd = List.of("touch", "-m", "--date=" + mDate, toFile.getPath());
x.execute(cmd);
} catch (Exception ignored) {}
}
}
}

22
src/main/java/com/olexyn/ensync/artifacts/SyncFile.java
Unescape View File

@ -1,7 +1,6 @@
package com.olexyn.ensync.artifacts;
import java.io.File;
import java.util.Map;
public class SyncFile extends File {
@ -16,7 +15,7 @@ public class SyncFile extends File {
super(pathname);
this.sd = sd;
relativePath = this.getPath().replace(sd.path, "");
relativePath = this.getPath().replace(sd.directoryPath, "");
}
public void setTimeModifiedFromStateFile(long value) {
@ -39,13 +38,26 @@ public class SyncFile extends File {
* If a File never existed, it will have time = 0, and thus will always be overwritten.
*/
public long getTimeModified(){
if (this.exists()){
if (exists()) {
return lastModified();
}
if (sd.readStateFile().get(this.getPath())!=null){
if (sd.readStateFile().get(getPath()) != null) {
return getTimeModifiedFromStateFile();
}
return 0;
return 0;
}
public boolean isNewer(SyncFile otherFile) {
return this.getTimeModified() >= otherFile.getTimeModified();
}
public boolean isOlder(SyncFile otherFile) {
return !isNewer(otherFile);
}
public SyncFile otherFile(SyncDirectory otherSd) {
return new SyncFile(otherSd, otherSd.directoryPath + this.relativePath);
}
}

10
src/test/test_cases.csv
Unescape View File

@ -0,0 +1,10 @@
Case
1
2
3
4
5
6
7
8
9
1 Case
2 1
3 2
4 3
5 4
6 5
7 6
8 7
9 8
10 9
Write Preview
Loading…
Cancel
Save