public
/
ensync
2
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

_ update logs

Browse Source 
_ update fc acquisition
_ add support for composite ignore entries
_ use buffer for copy and hash
pull/1/head
io42630 3 years ago
parent 670c096515
commit 6db86e0f5b
3 changed files with 63 additions and 38 deletions
Show Stats Download Patch File Download Diff File

11
src/main/java/com/olexyn/ensync/Flow.java
Unescape View File

@ -62,16 +62,17 @@ public class Flow implements Runnable {
private void sync(SyncDirectory sDir) {
LOGGER.info("DO SYNC " + sDir.directoryPath);
var listFileSystem = sDir.readFileSystem();
LOGGER.info("# of files on FS: " + listFileSystem.size());
LOGGER.info("# FS: " + listFileSystem.size());
var record = new Record(sDir.directoryPath);
record.getFiles().putAll(sDir.readRecord());
LOGGER.info("# of files on Record: " + record.getFiles().size());
LOGGER.info("# Record: " + record.getFiles().size());
var listCreated = sDir.fillListOfLocallyCreatedFiles(listFileSystem, record);
LOGGER.info("# of files in Created: " + listCreated.size());
LOGGER.info("# Created: " + listCreated.size());
var listDeleted = sDir.makeListOfLocallyDeletedFiles(listFileSystem, record);
LOGGER.info("# of files in Deleted: " + listDeleted.size());
LOGGER.info("# Deleted: " + listDeleted.size());
var listModified = sDir.makeListOfLocallyModifiedFiles(listFileSystem, record);
LOGGER.info("# of files in Modified: " + listModified.size());
LOGGER.info("# Modified: " + listModified.size());
sDir.doCreateOpsOnOtherSDs(listCreated);
sDir.doDeleteOpsOnOtherSDs(listDeleted);

82
src/main/java/com/olexyn/ensync/artifacts/SyncDirectory.java
Unescape View File

@ -4,29 +4,24 @@ import com.olexyn.ensync.LogUtil;
import com.olexyn.ensync.MainApp;
import com.olexyn.ensync.Tools;
import com.olexyn.ensync.lock.LockKeeper;
import com.olexyn.ensync.lock.LockUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.math.BigInteger;
import java.nio.channels.Channels;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@ -74,7 +69,7 @@ public class SyncDirectory {
Map<String, RecordFile> filemap = new HashMap<>();
var record = new Record(directoryPath);
var lines = tools.fileToLines(LockKeeper.getFc(record.getPath()));
var lines = Tools.fileToLines(LockKeeper.getFc(record.getPath()));
for (String line : lines) {
// this is a predefined format: "<modification-time>RECORD_SEPARATOR<relative-path>"
@ -85,7 +80,7 @@ public class SyncDirectory {
RecordFile recordFile = new RecordFile(this, absolutePath);
recordFile.setTimeModifiedFromRecord(modTime);
if (!shouldIgnore(recordFile.toPath())) {
if (noIgnore(recordFile.toPath())) {
filemap.put(sFilePath, recordFile);
}
}
@ -131,14 +126,17 @@ public class SyncDirectory {
private String getHash(Path path) {
var thisFc = LockKeeper.getFc(path);
byte[] data = Tools.fileToString(thisFc).getBytes(StandardCharsets.UTF_8);
try {
try (var is = Channels.newInputStream(thisFc)) {
var m = MessageDigest.getInstance("SHA256");
m.update(data, 0, data.length);
byte[] buffer = new byte[262144];
int bytesRead;
while ((bytesRead = is.read(buffer)) != -1) {
m.update(buffer, 0, bytesRead);
}
var i = new BigInteger(1, m.digest());
return String.format("%1$032X", i);
} catch (NoSuchAlgorithmException e) {
LOGGER.info("File not found.");
} catch (Exception e) {
LOGGER.log(Level.INFO, "Failed to create Hash.", e);
return null;
}
}
@ -165,13 +163,19 @@ public class SyncDirectory {
tools.writeStringListToFile(record.getPath().toString(), outputList);
}
private boolean shouldIgnore(Path path) {
for (var entry : MainApp.IGNORE) {
if (path.toString().contains(entry)) {
return true;
private boolean noIgnore(Path path) {
for (var line : MainApp.IGNORE) {
line = line
.replace("/", "")
.replace("\\", "");
var pathX = path.toString()
.replace("/", "")
.replace("\\", "");
if (pathX.contains(line)) {
return false;
}
}
return false;
return true;
}
private Stream<File> getFiles() {
@ -179,7 +183,7 @@ public class SyncDirectory {
return Files.walk(directoryPath)
.filter(Files::isRegularFile)
.map(Path::toFile)
.filter(file -> !shouldIgnore(file.toPath()))
.filter(file -> noIgnore(file.toPath()))
.filter(file -> !file.getName().equals(Constants.STATE_FILE_NAME));
} catch (IOException e) {
LOGGER.severe("Could walk the file tree : Record will be empty.");
@ -261,6 +265,7 @@ public class SyncDirectory {
private void dropAge(SyncFile thisFile, SyncFile otherFile) {
if (thisFile.lastModified() == otherFile.lastModified()) {
LOGGER.info("Same age, ignore");
return;
}
if (thisFile.lastModified() < otherFile.lastModified()) {
@ -273,19 +278,40 @@ public class SyncDirectory {
}
private void copyFile(SyncFile thisFile, SyncFile otherFile) {
if (!otherFile.getParentFile().exists()) {
try {
FileUtils.createParentDirectories(otherFile);
} catch (IOException e) {
LOGGER.info("Could not create Parent");
}
}
var thisFc = LockKeeper.getFc(thisFile.toPath());
var otherFc = LockKeeper.getFc(otherFile.toPath());
try (var br = Tools.reader(thisFc) ; var bw = Tools.writer(otherFc) ) {
IOUtils.copy(br, bw);
LOGGER.info(thisFile.toPath() + "lastModified before " + thisFile.lastModified());
LOGGER.info(otherFile.toPath() + "lastModified before " + otherFile.lastModified());
otherFile.setLastModified(thisFile.lastModified());
LOGGER.info(otherFile.toPath() + "lastModified before " + otherFile.lastModified());
} catch (IOException e) {
try (
var is = Channels.newInputStream(thisFc) ;
var os = Channels.newOutputStream(otherFc)
) {
copyStream(is, os);
} catch (Exception e) {
LOGGER.severe("Could not copy file from: " + thisFile.toPath());
LOGGER.severe(" to: " + otherFile.toPath());
e.printStackTrace();
}
LOGGER.info(thisFile.toPath() + " "+ thisFile.lastModified());
LOGGER.info(otherFile.toPath() + " " + otherFile.lastModified());
otherFile.setLastModified(thisFile.lastModified());
LOGGER.info(otherFile.toPath() + " " + otherFile.lastModified());
}
public static void copyStream(InputStream input, OutputStream output)
throws IOException
{
byte[] buffer = new byte[262144];
int bytesRead;
while ((bytesRead = input.read(buffer)) != -1)
{
output.write(buffer, 0, bytesRead);
}
}
}

8
src/main/java/com/olexyn/ensync/lock/LockKeeper.java
Unescape View File

@ -31,7 +31,6 @@ public class LockKeeper {
return false;
}
LOGGER.info("LOCKED " + fcStates.size() + " files in " + dirPath);
fcStates.forEach(fcState -> LOGGER.info(" " + fcState.getPath()));
fcStates.forEach(fcState -> LOCKS.put(fcState.getPath(), fcState));
return fcStates.stream().noneMatch(FcState::isUnlocked);
}
@ -46,11 +45,10 @@ public class LockKeeper {
}
public static FileChannel getFc(Path path) {
var fc = LOCKS.get(path).getFc();
if (fc != null && fc.isOpen()) {
return fc;
var fcState = LOCKS.get(path);
if (fcState != null && fcState.getFc() != null && fcState.getFc().isOpen()) {
return fcState.getFc();
}
FcState fcState;
if (!path.toFile().exists()) {
fcState = LockUtil.newFile(path);
} else {

Write Preview
Loading…
Cancel
Save