public
/
ensync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

+ replace log for each file op with result states that can be summarized or debugged

Browse Source
pull/1/merge
io42630 3 years ago
parent 4f9d9eb8d6
commit 83828c9853
4 changed files with 107 additions and 59 deletions
Show Stats Download Patch File Download Diff File

21
src/main/java/com/olexyn/ensync/Flow.java
Unescape View File

@ -1,12 +1,16 @@
package com.olexyn.ensync;
import com.olexyn.ensync.artifacts.DataRoot;
import com.olexyn.ensync.artifacts.OpsResultType;
import com.olexyn.ensync.artifacts.Record;
import com.olexyn.ensync.artifacts.SyncDirectory;
import com.olexyn.ensync.lock.LockKeeper;
import com.olexyn.ensync.util.FileMove;
import com.olexyn.ensync.util.TraceUtil;
import com.olexyn.min.log.LogU;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors;
@ -88,16 +92,23 @@ public class Flow implements Runnable {
LogU.infoPlain("# MODIFIED: %s", mapModified.size());
if (createdByMove == deletedByMove) {
LogU.infoPlain("(created by mv == deleted by mv) -> EXECUTE OPS");
sDir.doCreateOpsOnOtherSDs(mapCreated);
sDir.doDeleteOpsOnOtherSDs(mapDeleted);
sDir.doModifyOpsOnOtherSDs(mapModified);
LogU.infoPlain("RESULTS:");
var createResults = sDir.doCreateOpsOnOtherSDs(mapCreated);
printResults(createResults);
var deleteResults = sDir.doDeleteOpsOnOtherSDs(mapDeleted);
printResults(deleteResults);
var modifyResults = sDir.doModifyOpsOnOtherSDs(mapModified);
printResults(modifyResults);
sDir.writeRecord(record);
} else {
LogU.warnPlain("(created by mv != deleted by mv) -> ABORT");
}
}
private void printResults(Map<OpsResultType, List<FileMove>> map) {
map.entrySet().stream()
.filter(entry -> !entry.getValue().isEmpty())
.forEach(entry -> LogU.infoPlain("%-30s: %s", entry.getKey().name(), entry.getValue().size()));
}
/**

17
src/main/java/com/olexyn/ensync/artifacts/OpsResultType.java
Unescape View File

@ -0,0 +1,17 @@
package com.olexyn.ensync.artifacts;
public enum OpsResultType {
IGNORE_COPY_SAME_AGE,
IGNORE_COPY_OTHER_IS_NEWER,
IGNORE_COPY_NOT_A_FILE,
IGNORE_COPY_COULD_NOT_HASH,
IGNORE_DELETE_OTHER_NOT_FOUND,
IGNORE_DELETE_OTHER_IS_NEWER,
OK_DROPPED_AGE_FROM,
OK_DROPPED_AGE_TO,
OK_COPY,
OK_DELETE,
WARN_COULD_NOT_CREATE_PARENT,
WARN_COULD_NOT_COPY,
WARN_COULD_NOT_DELETE
}

113
src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java
Unescape View File

@ -2,6 +2,7 @@ package com.olexyn.ensync.artifacts;
import com.olexyn.ensync.Tools;
import com.olexyn.ensync.lock.LockKeeper;
import com.olexyn.ensync.util.FileMove;
import com.olexyn.ensync.util.HashUtil;
import com.olexyn.ensync.util.IgnoreUtil;
import com.olexyn.min.log.LogU;
@ -15,6 +16,7 @@ import java.nio.channels.Channels;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@ -25,6 +27,19 @@ import java.util.stream.Stream;
import static com.olexyn.ensync.artifacts.Constants.EMPTY;
import static com.olexyn.ensync.artifacts.Constants.RECORD_SEPARATOR;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_COPY_COULD_NOT_HASH;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_COPY_NOT_A_FILE;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_COPY_OTHER_IS_NEWER;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_COPY_SAME_AGE;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_DELETE_OTHER_IS_NEWER;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_DELETE_OTHER_NOT_FOUND;
import static com.olexyn.ensync.artifacts.OpsResultType.OK_COPY;
import static com.olexyn.ensync.artifacts.OpsResultType.OK_DELETE;
import static com.olexyn.ensync.artifacts.OpsResultType.OK_DROPPED_AGE_FROM;
import static com.olexyn.ensync.artifacts.OpsResultType.OK_DROPPED_AGE_TO;
import static com.olexyn.ensync.artifacts.OpsResultType.WARN_COULD_NOT_COPY;
import static com.olexyn.ensync.artifacts.OpsResultType.WARN_COULD_NOT_CREATE_PARENT;
import static com.olexyn.ensync.artifacts.OpsResultType.WARN_COULD_NOT_DELETE;
/**
* A SyncDirectory is a singular occurrence of a directory in the filesystems.
@ -156,31 +171,44 @@ public class SyncDirectory {
}
}
public void doCreateOpsOnOtherSDs(Map<String, SyncFile> listCreated) {
public Map<OpsResultType, List<FileMove>> doCreateOpsOnOtherSDs(Map<String, SyncFile> listCreated) {
var opsResults = initOpsResults();
for (var createdFile : listCreated.values()) {
for (var otherFile : otherFiles(createdFile)) {
writeFileIfNewer(createdFile, otherFile);
var result = writeFileIfNewer(createdFile, otherFile);
opsResults.get(result).add(new FileMove(createdFile.getAbsolutePath(), otherFile.getAbsolutePath()));
}
}
return opsResults;
}
public void doDeleteOpsOnOtherSDs(Map<String, SyncFile> listDeleted) {
int deleteCount = 0;
public Map<OpsResultType, List<FileMove>> doDeleteOpsOnOtherSDs(Map<String, SyncFile> listDeleted) {
var opsResults = initOpsResults();
for (var deletedFile : listDeleted.values()) {
for (var otherFile : otherFiles(deletedFile)) {
var ok = deleteFileIfNewer(deletedFile, otherFile);
if (ok) { deleteCount++; }
var result = deleteFileIfNewer(deletedFile, otherFile);
opsResults.get(result).add(new FileMove(deletedFile.getAbsolutePath(), otherFile.getAbsolutePath()));
}
}
LogU.infoPlain("DELETED " + deleteCount + "/" + listDeleted.size() + " Files.");
return opsResults;
}
public void doModifyOpsOnOtherSDs(Map<String, SyncFile> listModified) {
private Map<OpsResultType, List<FileMove>> initOpsResults() {
Map<OpsResultType, List<FileMove>> opsResults = new HashMap<>();
Arrays.stream(OpsResultType.values())
.forEach(opsResultType -> opsResults.put(opsResultType, new ArrayList<>()));
return opsResults;
}
public Map<OpsResultType, List<FileMove>> doModifyOpsOnOtherSDs(Map<String, SyncFile> listModified) {
var opsResults = initOpsResults();
for (var modifiedFile : listModified.values()) {
for (var otherFile : otherFiles(modifiedFile)) {
writeFileIfNewer(modifiedFile, otherFile);
var result = writeFileIfNewer(modifiedFile, otherFile);
opsResults.get(result).add(new FileMove(modifiedFile.getAbsolutePath(), otherFile.getAbsolutePath()));
}
}
return opsResults;
}
private Collection<SyncFile> otherFiles(SyncFile thisFile) {
@ -195,68 +223,53 @@ public class SyncDirectory {
* Here the >= is crucial, since otherFile might have == modified,
* but in that case we still want to delete both files.
*/
private boolean deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.exists()) {
LogU.infoPlain("Not deleted (not found) " + otherFile.toPath() + " not found.");
return false;
}
private OpsResultType deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.exists()) { return IGNORE_DELETE_OTHER_NOT_FOUND; }
if (thisFile.lastModified() >= otherFile.lastModified()) {
try {
Files.delete(otherFile.toPath());
LogU.infoPlain("Deleted " + otherFile.toPath());
return true;
return OK_DELETE;
} catch (IOException e) {
LogU.infoPlain("Not deleted (IOE) " + otherFile.toPath());
return false;
e.printStackTrace();
return WARN_COULD_NOT_DELETE;
}
}
LogU.infoPlain("Not deleted (other file modified recently)");
return false;
return IGNORE_DELETE_OTHER_IS_NEWER;
}
/**
* Overwrite other file if this file is newer.
*/
private void writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
LogU.infoPlain("Try write from: " + thisFile.toPath());
LogU.infoPlain(" to: " + otherFile.toPath());
if (!thisFile.isFile()) { return; }
private OpsResultType writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (!thisFile.isFile()) { return IGNORE_COPY_NOT_A_FILE; }
if (otherFile.exists()) {
var thisHash = HashUtil.getHash(thisFile.toPath());
var otherHash = HashUtil.getHash(otherFile.toPath());
if (thisHash == null || otherHash == null) { return; }
if (thisHash == null || otherHash == null) { return IGNORE_COPY_COULD_NOT_HASH; }
if (thisHash.equals(otherHash)) {
dropAge(thisFile, otherFile);
return;
return dropAge(thisFile, otherFile);
} else if (thisFile.lastModified() <= otherFile.lastModified()) {
LogU.infoPlain("Did not override due to target being newer.");
return;
return IGNORE_COPY_OTHER_IS_NEWER;
}
}
copyFile(thisFile, otherFile);
return copyFile(thisFile, otherFile);
}
private void dropAge(SyncFile thisFile, SyncFile otherFile) {
if (thisFile.lastModified() == otherFile.lastModified()) {
LogU.infoPlain("Same age, ignore");
return;
}
private OpsResultType dropAge(SyncFile thisFile, SyncFile otherFile) {
if (thisFile.lastModified() == otherFile.lastModified()) { return IGNORE_COPY_SAME_AGE; }
if (thisFile.lastModified() < otherFile.lastModified()) {
otherFile.setLastModified(thisFile.lastModified());
LogU.infoPlain("Dropped age of: %s -> %s",otherFile.toPath(), otherFile.lastModified());
return OK_DROPPED_AGE_TO;
} else {
thisFile.setLastModified(otherFile.lastModified());
LogU.infoPlain("Dropped age of: %s -> %s",thisFile.toPath(), thisFile.lastModified());
return OK_DROPPED_AGE_FROM;
}
}
private void copyFile(SyncFile thisFile, SyncFile otherFile) {
private OpsResultType copyFile(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.getParentFile().exists()) {
try {
FileUtils.createParentDirectories(otherFile);
} catch (IOException e) {
LogU.warnPlain("Could not create Parent");
}
try { FileUtils.createParentDirectories(otherFile); }
catch (IOException e) { return WARN_COULD_NOT_CREATE_PARENT; }
}
var thisFc = LockKeeper.getFc(thisFile.toPath());
var otherFc = LockKeeper.getFc(otherFile.toPath());
@ -266,24 +279,18 @@ public class SyncDirectory {
) {
copyStream(is, os);
} catch (Exception e) {
LogU.warnPlain("Could not copy file from: %s", thisFile.toPath());
LogU.warnPlain(" to: %s", otherFile.toPath());
e.printStackTrace();
return WARN_COULD_NOT_COPY;
}
LogU.infoPlain(thisFile.toPath() + " "+ thisFile.lastModified());
LogU.infoPlain(otherFile.toPath() + " " + otherFile.lastModified());
otherFile.setLastModified(thisFile.lastModified());
LogU.infoPlain(otherFile.toPath() + " " + otherFile.lastModified());
return OK_COPY;
}
public static void copyStream(InputStream input, OutputStream output)
throws IOException
{
throws IOException {
byte[] buffer = new byte[262144];
int bytesRead;
while ((bytesRead = input.read(buffer)) != -1)
{
while ((bytesRead = input.read(buffer)) != -1) {
output.write(buffer, 0, bytesRead);
}
}

13
src/main/java/com/olexyn/ensync/util/FileMove.java
Unescape View File

@ -0,0 +1,13 @@
package com.olexyn.ensync.util;
public class FileMove {
public String from;
public String to;
public FileMove(String from, String to) {
this.from = from;
this.to = to;
}
}
Write Preview
Loading…
Cancel
Save