+ replace log for each file op with result states that can be summarized or debugged

pull/1/merge
io42630 3 years ago
parent 4f9d9eb8d6
commit 83828c9853

@ -1,12 +1,16 @@
package com.olexyn.ensync; package com.olexyn.ensync;
import com.olexyn.ensync.artifacts.DataRoot; import com.olexyn.ensync.artifacts.DataRoot;
import com.olexyn.ensync.artifacts.OpsResultType;
import com.olexyn.ensync.artifacts.Record; import com.olexyn.ensync.artifacts.Record;
import com.olexyn.ensync.artifacts.SyncDirectory; import com.olexyn.ensync.artifacts.SyncDirectory;
import com.olexyn.ensync.lock.LockKeeper; import com.olexyn.ensync.lock.LockKeeper;
import com.olexyn.ensync.util.FileMove;
import com.olexyn.ensync.util.TraceUtil; import com.olexyn.ensync.util.TraceUtil;
import com.olexyn.min.log.LogU; import com.olexyn.min.log.LogU;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicBoolean;
import java.util.stream.Collectors; import java.util.stream.Collectors;
@ -88,16 +92,23 @@ public class Flow implements Runnable {
LogU.infoPlain("# MODIFIED: %s", mapModified.size()); LogU.infoPlain("# MODIFIED: %s", mapModified.size());
if (createdByMove == deletedByMove) { if (createdByMove == deletedByMove) {
LogU.infoPlain("(created by mv == deleted by mv) -> EXECUTE OPS"); LogU.infoPlain("(created by mv == deleted by mv) -> EXECUTE OPS");
sDir.doCreateOpsOnOtherSDs(mapCreated); LogU.infoPlain("RESULTS:");
sDir.doDeleteOpsOnOtherSDs(mapDeleted); var createResults = sDir.doCreateOpsOnOtherSDs(mapCreated);
sDir.doModifyOpsOnOtherSDs(mapModified); printResults(createResults);
var deleteResults = sDir.doDeleteOpsOnOtherSDs(mapDeleted);
printResults(deleteResults);
var modifyResults = sDir.doModifyOpsOnOtherSDs(mapModified);
printResults(modifyResults);
sDir.writeRecord(record); sDir.writeRecord(record);
} else { } else {
LogU.warnPlain("(created by mv != deleted by mv) -> ABORT"); LogU.warnPlain("(created by mv != deleted by mv) -> ABORT");
} }
}
private void printResults(Map<OpsResultType, List<FileMove>> map) {
map.entrySet().stream()
.filter(entry -> !entry.getValue().isEmpty())
.forEach(entry -> LogU.infoPlain("%-30s: %s", entry.getKey().name(), entry.getValue().size()));
} }
/** /**

@ -0,0 +1,17 @@
package com.olexyn.ensync.artifacts;
public enum OpsResultType {
IGNORE_COPY_SAME_AGE,
IGNORE_COPY_OTHER_IS_NEWER,
IGNORE_COPY_NOT_A_FILE,
IGNORE_COPY_COULD_NOT_HASH,
IGNORE_DELETE_OTHER_NOT_FOUND,
IGNORE_DELETE_OTHER_IS_NEWER,
OK_DROPPED_AGE_FROM,
OK_DROPPED_AGE_TO,
OK_COPY,
OK_DELETE,
WARN_COULD_NOT_CREATE_PARENT,
WARN_COULD_NOT_COPY,
WARN_COULD_NOT_DELETE
}

@ -2,6 +2,7 @@ package com.olexyn.ensync.artifacts;
import com.olexyn.ensync.Tools; import com.olexyn.ensync.Tools;
import com.olexyn.ensync.lock.LockKeeper; import com.olexyn.ensync.lock.LockKeeper;
import com.olexyn.ensync.util.FileMove;
import com.olexyn.ensync.util.HashUtil; import com.olexyn.ensync.util.HashUtil;
import com.olexyn.ensync.util.IgnoreUtil; import com.olexyn.ensync.util.IgnoreUtil;
import com.olexyn.min.log.LogU; import com.olexyn.min.log.LogU;
@ -15,6 +16,7 @@ import java.nio.channels.Channels;
import java.nio.file.Files; import java.nio.file.Files;
import java.nio.file.Path; import java.nio.file.Path;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection; import java.util.Collection;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@ -25,6 +27,19 @@ import java.util.stream.Stream;
import static com.olexyn.ensync.artifacts.Constants.EMPTY; import static com.olexyn.ensync.artifacts.Constants.EMPTY;
import static com.olexyn.ensync.artifacts.Constants.RECORD_SEPARATOR; import static com.olexyn.ensync.artifacts.Constants.RECORD_SEPARATOR;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_COPY_COULD_NOT_HASH;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_COPY_NOT_A_FILE;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_COPY_OTHER_IS_NEWER;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_COPY_SAME_AGE;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_DELETE_OTHER_IS_NEWER;
import static com.olexyn.ensync.artifacts.OpsResultType.IGNORE_DELETE_OTHER_NOT_FOUND;
import static com.olexyn.ensync.artifacts.OpsResultType.OK_COPY;
import static com.olexyn.ensync.artifacts.OpsResultType.OK_DELETE;
import static com.olexyn.ensync.artifacts.OpsResultType.OK_DROPPED_AGE_FROM;
import static com.olexyn.ensync.artifacts.OpsResultType.OK_DROPPED_AGE_TO;
import static com.olexyn.ensync.artifacts.OpsResultType.WARN_COULD_NOT_COPY;
import static com.olexyn.ensync.artifacts.OpsResultType.WARN_COULD_NOT_CREATE_PARENT;
import static com.olexyn.ensync.artifacts.OpsResultType.WARN_COULD_NOT_DELETE;
/** /**
* A SyncDirectory is a singular occurrence of a directory in the filesystems. * A SyncDirectory is a singular occurrence of a directory in the filesystems.
@ -156,31 +171,44 @@ public class SyncDirectory {
} }
} }
public void doCreateOpsOnOtherSDs(Map<String, SyncFile> listCreated) { public Map<OpsResultType, List<FileMove>> doCreateOpsOnOtherSDs(Map<String, SyncFile> listCreated) {
var opsResults = initOpsResults();
for (var createdFile : listCreated.values()) { for (var createdFile : listCreated.values()) {
for (var otherFile : otherFiles(createdFile)) { for (var otherFile : otherFiles(createdFile)) {
writeFileIfNewer(createdFile, otherFile); var result = writeFileIfNewer(createdFile, otherFile);
opsResults.get(result).add(new FileMove(createdFile.getAbsolutePath(), otherFile.getAbsolutePath()));
} }
} }
return opsResults;
} }
public void doDeleteOpsOnOtherSDs(Map<String, SyncFile> listDeleted) { public Map<OpsResultType, List<FileMove>> doDeleteOpsOnOtherSDs(Map<String, SyncFile> listDeleted) {
int deleteCount = 0; var opsResults = initOpsResults();
for (var deletedFile : listDeleted.values()) { for (var deletedFile : listDeleted.values()) {
for (var otherFile : otherFiles(deletedFile)) { for (var otherFile : otherFiles(deletedFile)) {
var ok = deleteFileIfNewer(deletedFile, otherFile); var result = deleteFileIfNewer(deletedFile, otherFile);
if (ok) { deleteCount++; } opsResults.get(result).add(new FileMove(deletedFile.getAbsolutePath(), otherFile.getAbsolutePath()));
} }
} }
LogU.infoPlain("DELETED " + deleteCount + "/" + listDeleted.size() + " Files."); return opsResults;
} }
public void doModifyOpsOnOtherSDs(Map<String, SyncFile> listModified) { private Map<OpsResultType, List<FileMove>> initOpsResults() {
Map<OpsResultType, List<FileMove>> opsResults = new HashMap<>();
Arrays.stream(OpsResultType.values())
.forEach(opsResultType -> opsResults.put(opsResultType, new ArrayList<>()));
return opsResults;
}
public Map<OpsResultType, List<FileMove>> doModifyOpsOnOtherSDs(Map<String, SyncFile> listModified) {
var opsResults = initOpsResults();
for (var modifiedFile : listModified.values()) { for (var modifiedFile : listModified.values()) {
for (var otherFile : otherFiles(modifiedFile)) { for (var otherFile : otherFiles(modifiedFile)) {
writeFileIfNewer(modifiedFile, otherFile); var result = writeFileIfNewer(modifiedFile, otherFile);
opsResults.get(result).add(new FileMove(modifiedFile.getAbsolutePath(), otherFile.getAbsolutePath()));
} }
} }
return opsResults;
} }
private Collection<SyncFile> otherFiles(SyncFile thisFile) { private Collection<SyncFile> otherFiles(SyncFile thisFile) {
@ -195,95 +223,74 @@ public class SyncDirectory {
* Here the >= is crucial, since otherFile might have == modified, * Here the >= is crucial, since otherFile might have == modified,
* but in that case we still want to delete both files. * but in that case we still want to delete both files.
*/ */
private boolean deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) { private OpsResultType deleteFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.exists()) { if (!otherFile.exists()) { return IGNORE_DELETE_OTHER_NOT_FOUND; }
LogU.infoPlain("Not deleted (not found) " + otherFile.toPath() + " not found.");
return false;
}
if (thisFile.lastModified() >= otherFile.lastModified()) { if (thisFile.lastModified() >= otherFile.lastModified()) {
try { try {
Files.delete(otherFile.toPath()); Files.delete(otherFile.toPath());
LogU.infoPlain("Deleted " + otherFile.toPath()); return OK_DELETE;
return true;
} catch (IOException e) { } catch (IOException e) {
LogU.infoPlain("Not deleted (IOE) " + otherFile.toPath()); e.printStackTrace();
return false; return WARN_COULD_NOT_DELETE;
} }
} }
LogU.infoPlain("Not deleted (other file modified recently)"); return IGNORE_DELETE_OTHER_IS_NEWER;
return false;
} }
/** /**
* Overwrite other file if this file is newer. * Overwrite other file if this file is newer.
*/ */
private void writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) { private OpsResultType writeFileIfNewer(SyncFile thisFile, SyncFile otherFile) {
LogU.infoPlain("Try write from: " + thisFile.toPath()); if (!thisFile.isFile()) { return IGNORE_COPY_NOT_A_FILE; }
LogU.infoPlain(" to: " + otherFile.toPath());
if (!thisFile.isFile()) { return; }
if (otherFile.exists()) { if (otherFile.exists()) {
var thisHash = HashUtil.getHash(thisFile.toPath()); var thisHash = HashUtil.getHash(thisFile.toPath());
var otherHash = HashUtil.getHash(otherFile.toPath()); var otherHash = HashUtil.getHash(otherFile.toPath());
if (thisHash == null || otherHash == null) { return; } if (thisHash == null || otherHash == null) { return IGNORE_COPY_COULD_NOT_HASH; }
if (thisHash.equals(otherHash)) { if (thisHash.equals(otherHash)) {
dropAge(thisFile, otherFile); return dropAge(thisFile, otherFile);
return;
} else if (thisFile.lastModified() <= otherFile.lastModified()) { } else if (thisFile.lastModified() <= otherFile.lastModified()) {
LogU.infoPlain("Did not override due to target being newer."); return IGNORE_COPY_OTHER_IS_NEWER;
return;
} }
} }
copyFile(thisFile, otherFile); return copyFile(thisFile, otherFile);
} }
private void dropAge(SyncFile thisFile, SyncFile otherFile) { private OpsResultType dropAge(SyncFile thisFile, SyncFile otherFile) {
if (thisFile.lastModified() == otherFile.lastModified()) { if (thisFile.lastModified() == otherFile.lastModified()) { return IGNORE_COPY_SAME_AGE; }
LogU.infoPlain("Same age, ignore");
return;
}
if (thisFile.lastModified() < otherFile.lastModified()) { if (thisFile.lastModified() < otherFile.lastModified()) {
otherFile.setLastModified(thisFile.lastModified()); otherFile.setLastModified(thisFile.lastModified());
LogU.infoPlain("Dropped age of: %s -> %s",otherFile.toPath(), otherFile.lastModified()); return OK_DROPPED_AGE_TO;
} else { } else {
thisFile.setLastModified(otherFile.lastModified()); thisFile.setLastModified(otherFile.lastModified());
LogU.infoPlain("Dropped age of: %s -> %s",thisFile.toPath(), thisFile.lastModified()); return OK_DROPPED_AGE_FROM;
} }
} }
private void copyFile(SyncFile thisFile, SyncFile otherFile) { private OpsResultType copyFile(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.getParentFile().exists()) { if (!otherFile.getParentFile().exists()) {
try { try { FileUtils.createParentDirectories(otherFile); }
FileUtils.createParentDirectories(otherFile); catch (IOException e) { return WARN_COULD_NOT_CREATE_PARENT; }
} catch (IOException e) {
LogU.warnPlain("Could not create Parent");
}
} }
var thisFc = LockKeeper.getFc(thisFile.toPath()); var thisFc = LockKeeper.getFc(thisFile.toPath());
var otherFc = LockKeeper.getFc(otherFile.toPath()); var otherFc = LockKeeper.getFc(otherFile.toPath());
try ( try (
var is = Channels.newInputStream(thisFc) ; var is = Channels.newInputStream(thisFc);
var os = Channels.newOutputStream(otherFc) var os = Channels.newOutputStream(otherFc)
) { ) {
copyStream(is, os); copyStream(is, os);
} catch (Exception e) { } catch (Exception e) {
LogU.warnPlain("Could not copy file from: %s", thisFile.toPath());
LogU.warnPlain(" to: %s", otherFile.toPath());
e.printStackTrace(); e.printStackTrace();
return WARN_COULD_NOT_COPY;
} }
LogU.infoPlain(thisFile.toPath() + " "+ thisFile.lastModified());
LogU.infoPlain(otherFile.toPath() + " " + otherFile.lastModified());
otherFile.setLastModified(thisFile.lastModified()); otherFile.setLastModified(thisFile.lastModified());
LogU.infoPlain(otherFile.toPath() + " " + otherFile.lastModified()); return OK_COPY;
} }
public static void copyStream(InputStream input, OutputStream output) public static void copyStream(InputStream input, OutputStream output)
throws IOException throws IOException {
{
byte[] buffer = new byte[262144]; byte[] buffer = new byte[262144];
int bytesRead; int bytesRead;
while ((bytesRead = input.read(buffer)) != -1) while ((bytesRead = input.read(buffer)) != -1) {
{
output.write(buffer, 0, bytesRead); output.write(buffer, 0, bytesRead);
} }
} }

@ -0,0 +1,13 @@
package com.olexyn.ensync.util;
public class FileMove {
public String from;
public String to;
public FileMove(String from, String to) {
this.from = from;
this.to = to;
}
}
Loading…
Cancel
Save