_ switch file ops to FileChannel

pull/1/head
io42630 3 years ago
parent c34e6d85aa
commit 8761dfdb50

@ -3,9 +3,11 @@ package com.olexyn.ensync;
import com.olexyn.ensync.artifacts.DataRoot;
import com.olexyn.ensync.artifacts.Record;
import com.olexyn.ensync.artifacts.SyncDirectory;
import com.olexyn.ensync.lock.LockKeeper;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.logging.Logger;
import java.util.stream.Collectors;
public class Flow implements Runnable {
@ -17,7 +19,7 @@ public class Flow implements Runnable {
public void start() {
LOGGER.info("START Flow.");
Thread worker = new Thread(this);
Thread worker = new Thread(this, "FLOW_WORKER");
worker.start();
}
@ -34,8 +36,15 @@ public class Flow implements Runnable {
DataRoot.getSyncBundles().forEach(
syncBundle -> {
var syncDirectories = syncBundle.getSyncDirectories();
var lockFail = syncDirectories.stream()
.map(sDir -> LockKeeper.lockDir(sDir.directoryPath))
.collect(Collectors.toList())
.contains(false);
if (!lockFail) {
syncDirectories.forEach(this::sync);
}
LockKeeper.unlockAll();
}
);
}
try {
@ -77,7 +86,7 @@ public class Flow implements Runnable {
*/
private void writeRecordIfMissing() {
DataRoot.get().values().forEach(syncBundle -> {
for (var sDir : syncBundle.syncDirectories.values()) {
for (var sDir : syncBundle.syncDirectories) {
var record = new Record(sDir.directoryPath);
if (!record.exists()) {
sDir.writeRecord(new Record(sDir.directoryPath));

@ -3,6 +3,7 @@ package com.olexyn.ensync;
import com.olexyn.ensync.artifacts.DataRoot;
import com.olexyn.ensync.artifacts.SyncBundle;
import com.olexyn.ensync.lock.LockUtil;
import org.json.JSONException;
import org.json.JSONObject;
@ -14,14 +15,14 @@ import java.util.List;
public class MainApp {
final public static Thread FLOW_THREAD = new Thread(new Flow(), "flow");
final public static Flow FLOW = new Flow();
public static List<String> IGNORE = new ArrayList<>();
final private static Tools tools = new Tools();
final private static Tools TOOLS = new Tools();
public static void main(String[] args) throws JSONException {
String configPath = System.getProperty("user.dir") + "/src/main/resources/config.json";
String configString = tools.fileToString(new File(configPath));
var configPath = Path.of(System.getProperty("user.dir") + "/src/main/resources/config.json");
String configString = Tools.fileToString(LockUtil.lockFile(configPath).getFc());
JSONObject dataRoot = new JSONObject(configString).getJSONObject("dataRoot");
for (String bundleKey : dataRoot.keySet()) {
SyncBundle syncBundle = new SyncBundle(bundleKey);
@ -33,9 +34,8 @@ public class MainApp {
DataRoot.get().put(bundleKey, syncBundle);
}
String ignorePath = System.getProperty("user.dir") + "/src/main/resources/syncignore";
IGNORE = tools.fileToLines(new File(ignorePath));
FLOW_THREAD.start();
var ignorePath = Path.of(System.getProperty("user.dir") + "/src/main/resources/syncignore");
IGNORE = Tools.fileToLines(LockUtil.lockFile(ignorePath).getFc());
FLOW.start();
}
}

@ -1,8 +1,12 @@
package com.olexyn.ensync;
import com.olexyn.ensync.artifacts.SyncFile;
import com.olexyn.ensync.lock.LockKeeper;
import java.io.*;
import java.nio.channels.Channels;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
@ -14,58 +18,31 @@ import java.util.Set;
public class Tools {
private final Execute x;
public Tools() {
x = new Execute();
public static BufferedReader reader(FileChannel fc) {
return new BufferedReader(Channels.newReader(fc, StandardCharsets.UTF_8));
}
/**
* Convert BufferedReader to String.
*
* @param br BufferedReader
* @return String
*/
public String brToString(BufferedReader br) {
StringBuilder sb = new StringBuilder();
Object[] br_array = br.lines().toArray();
for (int i = 0; i < br_array.length; i++) {
sb.append(br_array[i].toString() + "\n");
}
return sb.toString();
public static BufferedWriter writer(FileChannel fc) {
return new BufferedWriter(Channels.newWriter(fc, StandardCharsets.UTF_8));
}
/**
* Convert BufferedReader to List of Strings.
*
* @param br BufferedReader
* @return List
*/
public List<String> brToListString(BufferedReader br) {
List<String> list = new ArrayList<>();
Object[] br_array = br.lines().toArray();
for (int i = 0; i < br_array.length; i++) {
list.add(br_array[i].toString());
public static List<String> fileToLines(FileChannel fc) {
List<String> lines = new ArrayList<>();
try (var br = reader(fc)) {
String line;
while((line=br.readLine())!=null)
{
lines.add(line);
}
return list;
}
public List<String> fileToLines(File file) {
String filePath = file.getPath();
List<String> lines = null;
try {
lines = Files.readAllLines(Paths.get(filePath));
} catch (IOException e) {
e.printStackTrace();
}
return lines;
}
public String fileToString(File file){
List<String> lineList = fileToLines(file);
public static String fileToString(FileChannel fc){
var lineList = fileToLines(fc);
StringBuilder sb = new StringBuilder();
for (String line : lineList){
sb.append(line).append("\n");
@ -86,10 +63,9 @@ public class Tools {
public StringBuilder stringListToSb(List<String> list) {
StringBuilder sb = new StringBuilder();
var sb = new StringBuilder();
for (String line : list) {
sb.append(line + "\n");
sb.append(line).append("\n");
}
return sb;
}
@ -105,10 +81,8 @@ public class Tools {
}
public void writeSbToFile(File file, StringBuilder sb) {
try {
BufferedWriter bw = new BufferedWriter(new FileWriter(file));
try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))){
bw.write(sb.toString());
bw.close();
} catch (Exception e) {
e.printStackTrace();
}
@ -122,11 +96,9 @@ public class Tools {
* @param list <i>StringBuilder</i>
*/
public void writeStringListToFile(String path, List<String> list) {
try {
var bw = new BufferedWriter(new FileWriter(path));
try (var bw = new BufferedWriter(new FileWriter(path))) {
var sb = stringListToSb(list);
bw.write(sb.toString());
bw.close();
} catch (Exception e) {
e.printStackTrace();
}

@ -5,8 +5,10 @@ import com.olexyn.ensync.Tools;
import java.io.File;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
@ -16,7 +18,7 @@ import java.util.Map;
public class SyncBundle {
public String name;
public Map<Path, SyncDirectory> syncDirectories = new HashMap<>();
public List<SyncDirectory> syncDirectories = new ArrayList<>();
Tools tools = new Tools();
@ -28,7 +30,7 @@ public class SyncBundle {
}
public Collection<SyncDirectory> getSyncDirectories() {
return syncDirectories.values();
return syncDirectories;
}
/**
@ -40,12 +42,9 @@ public class SyncBundle {
*/
public void addDirectory(Path path) {
if (path.toFile().isDirectory()) {
syncDirectories.put(path, new SyncDirectory(path, this));
syncDirectories.add(new SyncDirectory(path, this));
}
}
public void removeDirectory(String realPath) {
syncDirectories.remove(realPath);
}
}

@ -3,12 +3,19 @@ package com.olexyn.ensync.artifacts;
import com.olexyn.ensync.LogUtil;
import com.olexyn.ensync.MainApp;
import com.olexyn.ensync.Tools;
import com.olexyn.ensync.lock.LockKeeper;
import com.olexyn.ensync.lock.LockUtil;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.math.BigInteger;
import java.nio.channels.Channels;
import java.nio.channels.ClosedChannelException;
import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
@ -35,9 +42,6 @@ public class SyncDirectory {
private static final Logger LOGGER = LogUtil.get(SyncDirectory.class);
private final SyncBundle syncMap;
public Path directoryPath;
public Map<String, SyncFile> listCreated = new HashMap<>();
public Map<String, SyncFile> listDeleted = new HashMap<>();
public Map<String, SyncFile> listModified = new HashMap<>();
Tools tools = new Tools();
/**
@ -69,7 +73,8 @@ public class SyncDirectory {
public Map<String, RecordFile> readRecord() {
Map<String, RecordFile> filemap = new HashMap<>();
var record = new Record(directoryPath);
List<String> lines = tools.fileToLines(record.getPath().toFile());
var lines = tools.fileToLines(LockKeeper.getFc(record.getPath()));
for (String line : lines) {
// this is a predefined format: "<modification-time>RECORD_SEPARATOR<relative-path>"
@ -125,13 +130,14 @@ public class SyncDirectory {
}
private String getHash(Path path) {
try (var fos = new FileInputStream(path.toFile())) {
var thisFc = LockKeeper.getFc(path);
byte[] data = Tools.fileToString(thisFc).getBytes(StandardCharsets.UTF_8);
try {
var m = MessageDigest.getInstance("SHA256");
byte[] data = fos.readAllBytes();
m.update(data, 0, data.length);
var i = new BigInteger(1, m.digest());
return String.format("%1$032X", i);
} catch (NoSuchAlgorithmException | IOException e) {
} catch (NoSuchAlgorithmException e) {
LOGGER.info("File not found.");
return null;
}
@ -218,13 +224,15 @@ public class SyncDirectory {
* but in that case we still want to delete both files.
*/
private void deleteFileIfNewer(RecordFile thisFile, SyncFile otherFile) {
if (!otherFile.exists()) { return; }
if (!otherFile.exists()) {
LOGGER.info("Could not delete: " + otherFile.toPath() + " not found.");
return; }
if (thisFile.lastModified() >= otherFile.lastModified()) {
try {
Files.delete(otherFile.toPath());
LOGGER.info("Deleted: " + otherFile.toPath());
} catch (IOException e) {
LOGGER.severe("Could not delete: " + otherFile.toPath());
LOGGER.info("Could not delete: " + otherFile.toPath());
}
}
}
@ -243,7 +251,7 @@ public class SyncDirectory {
if (thisHash.equals(otherHash)) {
dropAge(thisFile, otherFile);
return;
} else if (thisFile.isOlder(otherFile)) {
} else if (thisFile.lastModified() <= otherFile.lastModified()) {
LOGGER.info("Did not override due to target being newer.");
return;
}
@ -252,7 +260,10 @@ public class SyncDirectory {
}
private void dropAge(SyncFile thisFile, SyncFile otherFile) {
if (thisFile.isOlder(otherFile)) {
if (thisFile.lastModified() == otherFile.lastModified()) {
return;
}
if (thisFile.lastModified() < otherFile.lastModified()) {
otherFile.setLastModified(thisFile.lastModified());
LOGGER.info("Dropped age of: " + otherFile.toPath() + " -> " + otherFile.lastModified());
} else {
@ -262,15 +273,14 @@ public class SyncDirectory {
}
private void copyFile(SyncFile thisFile, SyncFile otherFile) {
try {
FileUtils.copyFile(
thisFile,
otherFile,
StandardCopyOption.REPLACE_EXISTING,
StandardCopyOption.COPY_ATTRIBUTES
);
LOGGER.info("Copied from: " + thisFile.toPath());
LOGGER.info(" to: " + otherFile.toPath());
var thisFc = LockKeeper.getFc(thisFile.toPath());
var otherFc = LockKeeper.getFc(otherFile.toPath());
try (var br = Tools.reader(thisFc) ; var bw = Tools.writer(otherFc) ) {
IOUtils.copy(br, bw);
LOGGER.info(thisFile.toPath() + "lastModified before " + thisFile.lastModified());
LOGGER.info(otherFile.toPath() + "lastModified before " + otherFile.lastModified());
otherFile.setLastModified(thisFile.lastModified());
LOGGER.info(otherFile.toPath() + "lastModified before " + otherFile.lastModified());
} catch (IOException e) {
LOGGER.severe("Could not copy file from: " + thisFile.toPath());
LOGGER.severe(" to: " + otherFile.toPath());

@ -41,13 +41,6 @@ public class SyncFile extends File {
return -1;
}
public boolean isNewer(SyncFile otherFile) {
return this.lastModified() >= otherFile.lastModified();
}
public boolean isOlder(SyncFile otherFile) {
return !isNewer(otherFile);
}
public SyncFile otherFile(SyncDirectory otherSd) {
return new SyncFile(otherSd, otherSd.directoryPath + this.relativePath);

@ -0,0 +1,34 @@
package com.olexyn.ensync.lock;
import java.nio.channels.FileChannel;
import java.nio.file.Path;
public class FcState {
Path path;
FileChannel fc;
boolean locked;
public FcState(Path path, FileChannel fc, boolean locked) {
this.path = path;
this.fc = fc;
this.locked = locked;
}
public FileChannel getFc() {
return fc;
}
public boolean isLocked() {
return locked;
}
public boolean isUnlocked() {
return !isLocked();
}
public Path getPath() {
return path;
}
}

@ -0,0 +1,63 @@
package com.olexyn.ensync.lock;
import com.olexyn.ensync.LogUtil;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.logging.Logger;
import java.util.stream.Collectors;
public class LockKeeper {
private static final int TRY_COUNT = 4;
private static final Logger LOGGER = LogUtil.get(LockKeeper.class);
private final static Map<Path, FcState> LOCKS = new HashMap<>();
public static boolean lockDir(Path dirPath) {
List<FcState> fcStates;
try {
fcStates = Files.walk(dirPath)
.filter(filePath -> filePath.toFile().isFile())
.map(filePath -> LockUtil.lockFile(filePath, TRY_COUNT))
.collect(Collectors.toList());
} catch (IOException e) {
return false;
}
LOGGER.info("LOCKED " + fcStates.size() + " files in " + dirPath);
fcStates.forEach(fcState -> LOGGER.info(" " + fcState.getPath()));
fcStates.forEach(fcState -> LOCKS.put(fcState.getPath(), fcState));
return fcStates.stream().noneMatch(FcState::isUnlocked);
}
public static void unlockAll() {
LOGGER.info("UNLOCKING ALL.");
LOCKS.values().forEach(
fcState -> LockUtil.unlockFile(fcState.getPath(), fcState.getFc(), 4)
);
}
public static FileChannel getFc(Path path) {
var fc = LOCKS.get(path).getFc();
if (fc != null && fc.isOpen()) {
return fc;
}
FcState fcState;
if (!path.toFile().exists()) {
fcState = LockUtil.newFile(path);
} else {
fcState = LockUtil.lockFile(path);
}
LOCKS.put(path, fcState);
return fcState.getFc();
}
}

@ -0,0 +1,80 @@
package com.olexyn.ensync.lock;
import com.olexyn.ensync.LogUtil;
import java.io.IOException;
import java.nio.channels.FileChannel;
import java.nio.channels.OverlappingFileLockException;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.logging.Level;
import java.util.logging.Logger;
import static java.nio.file.StandardOpenOption.APPEND;
import static java.nio.file.StandardOpenOption.CREATE_NEW;
import static java.nio.file.StandardOpenOption.READ;
import static java.nio.file.StandardOpenOption.WRITE;
import static java.util.logging.Level.INFO;
public class LockUtil {
private static final int DEFAULT_LOCK_TRIES = 4;
private static final long SLEEP_DURATION = 1000;
private static final Logger LOGGER = LogUtil.get(LockUtil.class);
public static FcState newFile(Path filePath) {
try {
var fc = FileChannel.open(filePath, CREATE_NEW, WRITE);
return new FcState(filePath, fc, false);
} catch (IOException | OverlappingFileLockException e) {
LOGGER.log(INFO, "Could not NEW " + filePath, e);
return new FcState(filePath, null, false);
}
}
public static FcState lockFile(Path filePath) {
return lockFile(filePath, DEFAULT_LOCK_TRIES);
}
public static FcState lockFile(Path filePath, int tryCount) {
try {
var fc = FileChannel.open(filePath, READ, WRITE);
if (filePath.toFile().exists()) {
fc.lock();
}
return new FcState(filePath, fc, true);
} catch (IOException | OverlappingFileLockException e) {
if (tryCount > 0) {
tryCount--;
LOGGER.info("Could not lock " + filePath + " Will try " + tryCount + " times.");
try {
Thread.sleep(SLEEP_DURATION);
} catch (InterruptedException ignored) { }
return lockFile(filePath, tryCount);
}
LOGGER.log(INFO, "Could not lock " + filePath, e);
return new FcState(filePath, null, false);
}
}
public static FcState unlockFile(FcState fcState, int tryCount) {
return unlockFile(fcState.getPath(), fcState.getFc(), tryCount);
}
public static FcState unlockFile(Path filePath, FileChannel fc, int tryCount) {
if (fc == null) { return null; }
try {
fc.close();
return new FcState(filePath, fc, false);
} catch (IOException | OverlappingFileLockException e) {
if (tryCount > 0) {
tryCount--;
LOGGER.info("Could not close " + fc + " Will try " + tryCount + " times.");
return unlockFile(filePath, fc, tryCount);
}
LOGGER.info("Could not unlock " + fc);
return new FcState(filePath, null, true);
}
}
}

@ -1,11 +1,12 @@
package com.olexyn.ensync.files;
import com.olexyn.ensync.Execute;
import com.olexyn.ensync.Flow;
import com.olexyn.ensync.LogUtil;
import com.olexyn.ensync.Tools;
import com.olexyn.ensync.artifacts.DataRoot;
import com.olexyn.ensync.artifacts.SyncBundle;
import com.olexyn.ensync.lock.LockUtil;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@ -21,8 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.logging.Logger;
import org.apache.commons.io.FileUtils;
/**
* Perform the 15 test cases in TestCases.xlsx.
@ -37,7 +36,7 @@ public class FifteenTests {
final private static Tools tools = new Tools();
private final static long M1000 = 600;
private final static long M1000 = 800;
private static final Path TEMP_DIR = Path.of(System.getProperty("user.dir") + "/src/test/temp");
DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_LOCAL_DATE_TIME;
@ -48,41 +47,41 @@ public class FifteenTests {
private final TestFile bFile = new TestFile(B_DIR + "/testfile.txt");
private List<String> createFile(File file) {
if (file.exists()) {
LOGGER.info("TEST can not create existing: " + file.toPath());
Assert.fail();
}
List<String> stringList = new ArrayList<>();
try {
stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " CREATED");
tools.writeStringListToFile(file.getAbsolutePath(), stringList);
LOGGER.info("TEST CREATE: " + file.toPath());
Thread.sleep(M1000);
} catch (InterruptedException e) {
System.out.println("");
}
return stringList;
}
private List<String> modifyFile(File file) {
List<String> stringList = new ArrayList<>();
try {
stringList.addAll(tools.fileToLines(file));
LOGGER.info("TEST TRY MODIFY: " + file.toPath());
var fcState = LockUtil.lockFile(file.toPath(), 10);
var stringList = new ArrayList<>(tools.fileToLines(fcState.getFc()));
stringList.add(LocalDateTime.now().format(dateTimeFormatter) + " MODIFIED");
tools.writeStringListToFile(file.getAbsolutePath(), stringList);
LOGGER.info("TEST MODIFY: " + file.toPath());
Thread.sleep(M1000);
} catch (InterruptedException e) {
System.out.println("");
}
LockUtil.unlockFile(fcState, 10);
LOGGER.info("TEST MODIFY UNLOCKED: " + file.toPath());
return stringList;
}
private static void deleteFile(File file) {
LOGGER.info("TEST TRY DELETE: " + file.toPath());
var fcState = LockUtil.lockFile(file.toPath(), 10);
try {
Files.delete(file.toPath());
LOGGER.info("TEST DELETE: " + file.toPath());
Thread.sleep(M1000);
} catch (IOException | InterruptedException e) {
} catch (IOException e) {
LOGGER.severe("Could not delete file." + file.toPath());
}
LockUtil.unlockFile(fcState, 10);
LOGGER.info("TEST DELETE UNLOCKED: " + file.toPath());
}
private void cleanDirs(Path... dirs) {
@ -235,6 +234,7 @@ public class FifteenTests {
FLOW.start();
waitX(M1000);
modifyFile(aFile);
waitX(M1000);
deleteFile(bFile);
waitX(M1000);
Assert.assertFalse(aFile.exists());
@ -248,6 +248,7 @@ public class FifteenTests {
FLOW.start();
waitX(M1000);
modifyFile(aFile);
waitX(M1000);
var bContent = createFile(bFile);
waitX(M1000);
Assert.assertEquals(bContent, aFile.readContent());

@ -1,16 +1,22 @@
package com.olexyn.ensync.files;
import com.olexyn.ensync.LogUtil;
import com.olexyn.ensync.Tools;
import com.olexyn.ensync.artifacts.SyncDirectory;
import com.olexyn.ensync.lock.LockKeeper;
import com.olexyn.ensync.lock.LockUtil;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.logging.Logger;
public class TestFile extends File {
private static final Logger LOGGER = LogUtil.get(TestFile.class);
Tools tools = new Tools();
private List<String> content = new ArrayList<>();
/**
* Wrapper for File that adds tools for assessing it's state.
@ -19,28 +25,13 @@ public class TestFile extends File {
super(pathname);
}
public void setContent(List<String> content) {
this.content = content;
}
public List<String> readContent() {
this.content = tools.fileToLines(this);
return content;
LOGGER.info("TEST TRY READ: " + toPath());
var fcState = LockUtil.lockFile(toPath());
return tools.fileToLines(fcState.getFc());
}
public List<String> getContent() {
return content;
}
public List<String> copyContent() {
return List.copyOf(content);
}
public TestFile updateContent() {
String line = tools.fileToLines(this).get(0);
this.content.add(line);
return this;
}
@Override
public boolean equals(Object o) {

Loading…
Cancel
Save