summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorLudovic Pouzenc <ludovic@pouzenc.fr>2014-09-20 09:17:18 +0200
committerLudovic Pouzenc <ludovic@pouzenc.fr>2015-04-14 07:44:29 +0200
commitd6f22a2af48f83d63b5381118d2029797458194e (patch)
treecb6bef9a98335a7af2aee40b0752d14fcee0916e /src
parent774194091e9bcee08e48fcdf4127f9afd9d6d644 (diff)
downloadsssync-d6f22a2af48f83d63b5381118d2029797458194e.tar.gz
sssync-d6f22a2af48f83d63b5381118d2029797458194e.tar.bz2
sssync-d6f22a2af48f83d63b5381118d2029797458194e.zip
Early development stages (before SCM) : WIP_1
Early development stages (before SCM) : WIP_2 Early development stages (before SCM) : WIP_3 Early development stages (before SCM) : WIP_4 Early development stages (before SCM) : WIP_6 Early development stages (before SCM) : WIP_7 Early development stages (before SCM) : WIP_8 Adds documentation folder as an Eclipse project. Adds README for github. Decent source tree by tuning Eclise project's location One forgetten file while movign everything :) Adding Copyright, licencing (GPL v3), correcting README
Diffstat (limited to 'src')
-rw-r--r--src/connectors/.classpath21
-rw-r--r--src/connectors/.project17
-rw-r--r--src/connectors/.settings/org.eclipse.jdt.core.prefs11
-rw-r--r--src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java50
-rw-r--r--src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java94
-rw-r--r--src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java16
-rw-r--r--src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java115
-rw-r--r--src/connectors/JUTests/data/io/sql/req_test.sql5
-rw-r--r--src/connectors/build.xml89
-rw-r--r--src/connectors/lib/commons-csv-1.0-SNAPSHOT.jarbin0 -> 34791 bytes
-rw-r--r--src/connectors/lib/derby.jarbin0 -> 2838580 bytes
-rw-r--r--src/connectors/lib/derbytools.jarbin0 -> 214415 bytes
-rw-r--r--src/connectors/lib/mysql-connector-java-5.1.31-bin.jarbin0 -> 964882 bytes
-rw-r--r--src/connectors/lib/ojdbc6-javadoc.jarbin0 -> 831616 bytes
-rw-r--r--src/connectors/lib/ojdbc6.jarbin0 -> 2739670 bytes
-rw-r--r--src/connectors/lib/orai18n.jarbin0 -> 1655734 bytes
-rw-r--r--src/connectors/lib/unboundid-ldapsdk-se-javadoc.jarbin0 -> 5214452 bytes
-rw-r--r--src/connectors/lib/unboundid-ldapsdk-se.jarbin0 -> 1556717 bytes
-rw-r--r--src/connectors/src/data/io/csv/CSVDataReader.java248
-rw-r--r--src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java112
-rw-r--r--src/connectors/src/data/io/ldap/LDAPFlatDataReader.java178
-rw-r--r--src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java198
-rw-r--r--src/connectors/src/data/io/sql/SQLConnectionWrapper.java136
-rw-r--r--src/connectors/src/data/io/sql/SQLRelDataReader.java173
-rw-r--r--src/core/.classpath9
-rw-r--r--src/core/.project17
-rw-r--r--src/core/.settings/org.eclipse.jdt.core.prefs11
-rw-r--r--src/core/JUTests/data/MVDataEntryTest.java93
-rw-r--r--src/core/JUTests/data/io/filters/MVDataCombinerTest.java148
-rw-r--r--src/core/build.xml78
-rw-r--r--src/core/lib/guava-16.0.1.jarbin0 -> 2228009 bytes
-rw-r--r--src/core/src/data/MVDataEntry.java238
-rw-r--r--src/core/src/data/filters/MVDataCombiner.java164
-rw-r--r--src/core/src/data/io/AbstractMVDataReader.java49
-rw-r--r--src/core/src/data/io/AbstractMVDataWriter.java70
-rw-r--r--src/core/src/data/io/MVDataReader.java39
-rw-r--r--src/core/src/data/io/MVDataWriter.java45
-rw-r--r--src/core/src/data/io/stub/StubDataReader.java63
-rw-r--r--src/core/src/data/io/stub/StubDataWriter.java104
-rw-r--r--src/core/src/sync/AbstractSyncTask.java71
-rw-r--r--src/main/.classpath16
-rw-r--r--src/main/.project17
-rw-r--r--src/main/.settings/org.eclipse.jdt.core.prefs11
-rw-r--r--src/main/JUTests/AllClientServerTests.java18
-rw-r--r--src/main/JUTests/AllLocalTests.java29
-rw-r--r--src/main/JUTests/conf/SSSyncConfParserTest.java69
-rw-r--r--src/main/JUTests/conf/testConn.yaml19
-rw-r--r--src/main/JUTests/conf/testExpectedConn.yaml22
-rw-r--r--src/main/JUTests/conf/testExpectedMain.yaml70
-rw-r--r--src/main/JUTests/conf/testMain.yaml54
-rw-r--r--src/main/JUTests/data/io/SafeDataReaderTest.java51
-rw-r--r--src/main/JUTests/sync/BasicSyncTaskTest.java129
-rw-r--r--src/main/build.xml101
-rw-r--r--src/main/conf/connections.yaml18
-rw-r--r--src/main/conf/log4j.properties29
-rw-r--r--src/main/conf/queries/people.sql5
-rw-r--r--src/main/conf/queries/structures.sql5
-rw-r--r--src/main/conf/sssync.yaml56
-rw-r--r--src/main/data/people_append.csv1
-rw-r--r--src/main/data/people_replace.csv3
-rw-r--r--src/main/lib/log4j-1.2.17.jarbin0 -> 489883 bytes
-rw-r--r--src/main/lib/snakeyaml-1.11-javadoc.jarbin0 -> 766655 bytes
-rw-r--r--src/main/lib/snakeyaml-1.11.jarbin0 -> 270552 bytes
-rw-r--r--src/main/src/SSSync.java208
-rw-r--r--src/main/src/conf/ConfigConnectionBean.java111
-rw-r--r--src/main/src/conf/ConfigConnectionsBean.java45
-rw-r--r--src/main/src/conf/ConfigGlobalsBean.java41
-rw-r--r--src/main/src/conf/ConfigOpLimitsBean.java55
-rw-r--r--src/main/src/conf/ConfigRootBean.java73
-rw-r--r--src/main/src/conf/ConfigSrcOrDestBean.java96
-rw-r--r--src/main/src/conf/ConfigTaskBean.java80
-rw-r--r--src/main/src/conf/SSSyncConfParser.java65
-rw-r--r--src/main/src/conf/SSSyncConnectionsFactory.java61
-rw-r--r--src/main/src/conf/SSSyncTasksFactory.java147
-rw-r--r--src/main/src/data/io/ConnectionsHolder.java81
-rw-r--r--src/main/src/data/io/SafeDataReader.java155
-rw-r--r--src/main/src/sync/BasicSyncTask.java292
-rw-r--r--src/main/src/utils/JVMStatsDumper.java111
-rwxr-xr-xsrc/main/sssync.sh3
-rwxr-xr-xsrc/sloc.sh7
80 files changed, 5016 insertions, 0 deletions
diff --git a/src/connectors/.classpath b/src/connectors/.classpath
new file mode 100644
index 0000000..e421f8f
--- /dev/null
+++ b/src/connectors/.classpath
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="src" path="JUTests"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/SSSync_Core"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
+ <classpathentry kind="lib" path="lib/commons-csv-1.0-SNAPSHOT.jar"/>
+ <classpathentry kind="lib" path="lib/ojdbc6.jar">
+ <attributes>
+ <attribute name="javadoc_location" value="jar:platform:/resource/SSSync_Connectors/lib/ojdbc6-javadoc.jar!/"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="lib" path="lib/mysql-connector-java-5.1.31-bin.jar"/>
+ <classpathentry kind="lib" path="lib/unboundid-ldapsdk-se.jar">
+ <attributes>
+ <attribute name="javadoc_location" value="jar:platform:/resource/SSSync_Connectors/lib/unboundid-ldapsdk-se-javadoc.jar!/"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
diff --git a/src/connectors/.project b/src/connectors/.project
new file mode 100644
index 0000000..b4f50df
--- /dev/null
+++ b/src/connectors/.project
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>SSSync_Connectors</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
diff --git a/src/connectors/.settings/org.eclipse.jdt.core.prefs b/src/connectors/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..8000cd6
--- /dev/null
+++ b/src/connectors/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,11 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.6
diff --git a/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java b/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java
new file mode 100644
index 0000000..6a0e053
--- /dev/null
+++ b/src/connectors/JUTests/data/io/csv/CSVDataReaderTest.java
@@ -0,0 +1,50 @@
+package data.io.csv;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Iterator;
+
+import org.junit.Test;
+
+import data.MVDataEntry;
+
+public class CSVDataReaderTest {
+
+
+ @Test
+ public void testNext() throws IOException {
+ CSVDataReader reader = new CSVDataReader(
+ "testNext",
+ new StringReader(CSVDataReader.CSV_DEMO),
+ false
+ );
+
+ MVDataEntry expected[] = new MVDataEntry[3];
+ expected[0]=new MVDataEntry("line1");
+ expected[0].splitAndPut("from", "csv1;csv1bis", ";");
+ expected[0].splitAndPut("attr2","csv1",";");
+
+ expected[1]=new MVDataEntry("line2");
+ expected[1].splitAndPut("hello", "all;the;world", ";");
+
+ expected[2]=new MVDataEntry("line3");
+ expected[2].splitAndPut("hello", "all;the;others", ";");
+
+ // Test twice to check if asking a new iterator "rewinds" correctly
+ for (int i=0;i<2;i++) {
+ System.out.println("Loop " + (i+1));
+ Iterator<MVDataEntry> readerIt = reader.iterator();
+
+ for ( MVDataEntry e: expected) {
+ assertTrue(readerIt.hasNext());
+ MVDataEntry r = readerIt.next();
+ System.out.println(e + " / " + r);
+ assertEquals(e, r);
+ }
+ assertFalse(readerIt.hasNext());
+ }
+ }
+
+}
diff --git a/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java b/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java
new file mode 100644
index 0000000..dcfc602
--- /dev/null
+++ b/src/connectors/JUTests/data/io/ldap/LDAPDataReaderTest.java
@@ -0,0 +1,94 @@
+package data.io.ldap;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import data.MVDataEntry;
+
+public class LDAPDataReaderTest {
+
+ LDAPConnectionWrapper builder;
+
+ @Before
+ public void setup() {
+ builder = new LDAPConnectionWrapper("localhost", 389, "uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr", "secret");
+ }
+
+ /*
+ @Test
+ public void testLookAhead1() {
+ _testLookAhead(1);
+ }
+ */
+
+ @Test
+ public void testLookAhead16() {
+ _testLookAhead(16);
+ }
+
+ @Test
+ public void testLookAhead32() {
+ _testLookAhead(32);
+ }
+
+ @Test
+ public void testLookAhead64() {
+ _testLookAhead(64);
+ }
+
+ @Test
+ public void testLookAhead128() {
+ _testLookAhead(128);
+ }
+
+ @Test
+ public void testLookAhead192() {
+ _testLookAhead(192);
+ }
+
+ @Test
+ public void testLookAhead256() {
+ _testLookAhead(256);
+ }
+
+ @Test
+ public void testLookAhead512() {
+ _testLookAhead(512);
+ }
+
+ @Test
+ public void testLookAhead1024() {
+ _testLookAhead(1024);
+ }
+
+ private void _testLookAhead(int lookAheadAmount) {
+ System.out.println("_testLookAhead("+lookAheadAmount+")");
+ LDAPFlatDataReader reader = builder.newFlatReader("ldap_test", "ou=people,dc=univ-jfc,dc=fr", "uid", lookAheadAmount);
+
+ int resultCount = 0;
+ String previousKey=null;
+ for ( MVDataEntry entry : reader ) {
+ //System.out.println(entry);
+ if ( previousKey != null ) assertTrue(entry.getKey().compareTo(previousKey) > 0);
+ resultCount++;
+ previousKey=entry.getKey();
+ }
+ System.out.println(resultCount);
+ assertTrue(resultCount>0);
+
+ // Second time with a second iterator (must give the same results)
+ int newResultCount = 0;
+ previousKey=null;
+ for ( MVDataEntry entry : reader ) {
+ //System.out.println(entry);
+ if ( previousKey != null ) assertTrue(entry.getKey().compareTo(previousKey) > 0);
+ newResultCount++;
+ previousKey=entry.getKey();
+ }
+ System.out.println(newResultCount);
+ assertTrue(newResultCount == resultCount);
+
+ }
+}
diff --git a/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java b/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java
new file mode 100644
index 0000000..01a8af0
--- /dev/null
+++ b/src/connectors/JUTests/data/io/ldap/LDAPDataWriterTest.java
@@ -0,0 +1,16 @@
+package data.io.ldap;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class LDAPDataWriterTest {
+
+ @Test
+ public void test() {
+ fail("Not yet implemented");
+ }
+
+ // TODO : test update() extensively : null, empty string, add/update/delete subcases...
+
+}
diff --git a/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java b/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java
new file mode 100644
index 0000000..a97a98d
--- /dev/null
+++ b/src/connectors/JUTests/data/io/sql/SQLRelDataReaderTest.java
@@ -0,0 +1,115 @@
+package data.io.sql;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import data.MVDataEntry;
+import data.io.MVDataReader;
+import data.io.sql.SQLConnectionWrapper.DBMSType;
+
+
+/*
+
+CREATE TABLE sssync.people (
+ uid CHAR(16) NULL ,
+ uidNumber INT NOT NULL ,
+ gidNumber INT NULL ,
+ cn VARCHAR(45) NULL ,
+ sn VARCHAR(45) NULL ,
+ homeDirectory VARCHAR(45) NULL ,
+ PRIMARY KEY (uid) );
+INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('lpouzenc', 1000, 999, 'Ludovic', 'Pouzenc', '/home/lpouzenc');
+INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('dpouzenc', 1001, 999, 'Daniel', 'Pouzenc', '/home/dpouzenc');
+
+
+for i in $(seq 10000 20000); do echo "INSERT INTO sssync.people (uid, uidNumber, gidNumber, cn, sn, homeDirectory) VALUES ('test$i', $i, 999, '$i', 'test', '/home/test$i');"; done | mysql -uroot -p
+
+
+
+DROP TABLE IF EXISTS structures;
+CREATE TABLE structures (
+ supannCodeEntite varchar(15) NOT NULL,
+ ou varchar(45) NOT NULL,
+ supannTypeEntite varchar(45) NOT NULL,
+ supannCodeEntiteParent varchar(45) NOT NULL,
+ PRIMARY KEY (supannCodeEntite)
+) ENGINE=InnoDB DEFAULT CHARSET=latin1;
+
+INSERT INTO structures VALUES ('2','CUFR','Etablissement','2'),('9','Personnels','Groupe','2');
+
+
+
+TODO : make automated tests with embded Derby base
+
+ */
+public class SQLRelDataReaderTest {
+
+ private static final String TEST_REQUEST = "SELECT p.*, \"person;posixAccount;top\" as objectClass" +
+ " FROM sssync.people p" +
+ " ORDER BY 1 ASC;";
+
+ private SQLConnectionWrapper builder;
+ private MVDataReader reader1;
+ private MVDataReader reader2;
+
+ @Before
+ public void setup() throws IOException {
+ // Find the folder containing this test class
+ URL main = SQLRelDataReaderTest.class.getResource("SQLRelDataReaderTest.class");
+ if (!"file".equalsIgnoreCase(main.getProtocol()))
+ throw new IllegalStateException("This class is not stored in a file");
+ File currentFolder = new File(main.getPath()).getParentFile();
+
+ // Build a connection and two readers on it
+ builder = new SQLConnectionWrapper(DBMSType.mysql, "localhost", 3306, null, "root", "secret", "sssync");
+ reader1 = builder.newReader("testMysql1", TEST_REQUEST);
+ reader2 = builder.newReader("testMysql2", new File(currentFolder, "req_test.sql"));
+ }
+
+ @Test
+ public void testNext() {
+ // First full read on reader1
+ int resultCount_r1i1 = 0;
+ String previousKey_r1i1=null;
+ for ( MVDataEntry entry : reader1 ) {
+ //System.out.println(entry);
+ if ( previousKey_r1i1 != null ) assertTrue(entry.getKey().compareTo(previousKey_r1i1) > 0);
+ resultCount_r1i1++;
+ previousKey_r1i1=entry.getKey();
+ }
+ System.out.println(resultCount_r1i1);
+ assertTrue(resultCount_r1i1 > 0);
+
+ // First half read on reader2
+ int resultCount_r2i1 = 0;
+ String previousKey_r2i1=null;
+ for ( MVDataEntry entry : reader2 ) {
+ //System.out.println(entry);
+ if ( previousKey_r2i1 != null ) assertTrue(entry.getKey().compareTo(previousKey_r2i1) > 0);
+ resultCount_r2i1++;
+ previousKey_r2i1=entry.getKey();
+ if ( resultCount_r2i1 > resultCount_r1i1 / 2 ) break;
+ }
+ System.out.println(resultCount_r2i1);
+ assertTrue(resultCount_r2i1 > resultCount_r1i1 / 2 );
+
+ // Second time with a second iterator on reader1 (must give the same results than r1i1)
+ int resultCount_r1i2 = 0;
+ String previousKey_r1i2=null;
+ for ( MVDataEntry entry : reader1 ) {
+ //System.out.println(entry);
+ if ( previousKey_r1i2 != null ) assertTrue(entry.getKey().compareTo(previousKey_r1i2) > 0);
+ resultCount_r1i2++;
+ previousKey_r1i2=entry.getKey();
+ }
+ System.out.println(resultCount_r1i2);
+ assertTrue(resultCount_r1i2 == resultCount_r1i1);
+ }
+
+}
diff --git a/src/connectors/JUTests/data/io/sql/req_test.sql b/src/connectors/JUTests/data/io/sql/req_test.sql
new file mode 100644
index 0000000..ab66d5f
--- /dev/null
+++ b/src/connectors/JUTests/data/io/sql/req_test.sql
@@ -0,0 +1,5 @@
+SELECT
+ p.*,
+ "person;posixAccount;top" as objectClass
+FROM sssync.people p
+ORDER BY 1 ASC;
diff --git a/src/connectors/build.xml b/src/connectors/build.xml
new file mode 100644
index 0000000..fdae9de
--- /dev/null
+++ b/src/connectors/build.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- WARNING: Eclipse auto-generated file.
+ Any modifications will be overwritten.
+ To include a user specific buildfile here, simply create one in the same
+ directory with the processing instruction <?eclipse.ant.import?>
+ as the first entry and export the buildfile again. -->
+<project basedir="." default="build" name="SSSync_Connectors">
+ <property environment="env"/>
+ <property name="SSSync_Main.location" value="../main"/>
+ <property name="ECLIPSE_HOME" value="../../../../../../usr/lib/eclipse"/>
+ <property name="SSSync_Core.location" value="../core"/>
+ <property name="debuglevel" value="source,lines,vars"/>
+ <property name="target" value="1.6"/>
+ <property name="source" value="1.6"/>
+ <path id="JUnit 4.libraryclasspath">
+ <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.junit_4.8.2.dist/junit.jar"/>
+ <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.hamcrest.core_1.1.0.jar"/>
+ </path>
+ <path id="SSSync_Core.classpath">
+ <pathelement location="${SSSync_Core.location}/bin"/>
+ <pathelement location="${SSSync_Core.location}/lib/guava-16.0.1.jar"/>
+ <path refid="JUnit 4.libraryclasspath"/>
+ </path>
+ <path id="SSSync_Connectors.classpath">
+ <pathelement location="bin"/>
+ <path refid="SSSync_Core.classpath"/>
+ <path refid="JUnit 4.libraryclasspath"/>
+ <pathelement location="lib/commons-csv-1.0-SNAPSHOT.jar"/>
+ <pathelement location="lib/ojdbc6.jar"/>
+ <pathelement location="lib/mysql-connector-java-5.1.31-bin.jar"/>
+ <pathelement location="lib/unboundid-ldapsdk-se.jar"/>
+ </path>
+ <target name="init">
+ <mkdir dir="bin"/>
+ <copy includeemptydirs="false" todir="bin">
+ <fileset dir="src">
+ <exclude name="**/*.java"/>
+ </fileset>
+ </copy>
+ <copy includeemptydirs="false" todir="bin">
+ <fileset dir="JUTests">
+ <exclude name="**/*.java"/>
+ </fileset>
+ </copy>
+ </target>
+ <target name="clean">
+ <delete dir="bin"/>
+ </target>
+ <target depends="clean" name="cleanall">
+ <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="clean"/>
+ </target>
+ <target depends="build-subprojects,build-project" name="build"/>
+ <target name="build-subprojects">
+ <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="build-project">
+ <propertyset>
+ <propertyref name="build.compiler"/>
+ </propertyset>
+ </ant>
+ </target>
+ <target depends="init" name="build-project">
+ <echo message="${ant.project.name}: ${ant.file}"/>
+ <javac debug="true" debuglevel="${debuglevel}" destdir="bin" includeantruntime="false" source="${source}" target="${target}">
+ <src path="src"/>
+ <src path="JUTests"/>
+ <classpath refid="SSSync_Connectors.classpath"/>
+ </javac>
+ </target>
+ <target description="Build all projects which reference this project. Useful to propagate changes." name="build-refprojects">
+ <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="clean"/>
+ <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="build">
+ <propertyset>
+ <propertyref name="build.compiler"/>
+ </propertyset>
+ </ant>
+ </target>
+ <target description="copy Eclipse compiler jars to ant lib directory" name="init-eclipse-compiler">
+ <copy todir="${ant.library.dir}">
+ <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/>
+ </copy>
+ <unzip dest="${ant.library.dir}">
+ <patternset includes="jdtCompilerAdapter.jar"/>
+ <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/>
+ </unzip>
+ </target>
+ <target description="compile project with Eclipse compiler" name="build-eclipse-compiler">
+ <property name="build.compiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/>
+ <antcall target="build"/>
+ </target>
+</project>
diff --git a/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar b/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar
new file mode 100644
index 0000000..f6a74f1
--- /dev/null
+++ b/src/connectors/lib/commons-csv-1.0-SNAPSHOT.jar
Binary files differ
diff --git a/src/connectors/lib/derby.jar b/src/connectors/lib/derby.jar
new file mode 100644
index 0000000..a4d56f0
--- /dev/null
+++ b/src/connectors/lib/derby.jar
Binary files differ
diff --git a/src/connectors/lib/derbytools.jar b/src/connectors/lib/derbytools.jar
new file mode 100644
index 0000000..216ff3e
--- /dev/null
+++ b/src/connectors/lib/derbytools.jar
Binary files differ
diff --git a/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar b/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar
new file mode 100644
index 0000000..85ae51d
--- /dev/null
+++ b/src/connectors/lib/mysql-connector-java-5.1.31-bin.jar
Binary files differ
diff --git a/src/connectors/lib/ojdbc6-javadoc.jar b/src/connectors/lib/ojdbc6-javadoc.jar
new file mode 100644
index 0000000..81dfb08
--- /dev/null
+++ b/src/connectors/lib/ojdbc6-javadoc.jar
Binary files differ
diff --git a/src/connectors/lib/ojdbc6.jar b/src/connectors/lib/ojdbc6.jar
new file mode 100644
index 0000000..767eba7
--- /dev/null
+++ b/src/connectors/lib/ojdbc6.jar
Binary files differ
diff --git a/src/connectors/lib/orai18n.jar b/src/connectors/lib/orai18n.jar
new file mode 100644
index 0000000..9fad382
--- /dev/null
+++ b/src/connectors/lib/orai18n.jar
Binary files differ
diff --git a/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar b/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar
new file mode 100644
index 0000000..b724779
--- /dev/null
+++ b/src/connectors/lib/unboundid-ldapsdk-se-javadoc.jar
Binary files differ
diff --git a/src/connectors/lib/unboundid-ldapsdk-se.jar b/src/connectors/lib/unboundid-ldapsdk-se.jar
new file mode 100644
index 0000000..0932139
--- /dev/null
+++ b/src/connectors/lib/unboundid-ldapsdk-se.jar
Binary files differ
diff --git a/src/connectors/src/data/io/csv/CSVDataReader.java b/src/connectors/src/data/io/csv/CSVDataReader.java
new file mode 100644
index 0000000..6dbc8ff
--- /dev/null
+++ b/src/connectors/src/data/io/csv/CSVDataReader.java
@@ -0,0 +1,248 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+package data.io.csv;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.SortedSet;
+import java.util.TreeSet;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+
+import data.MVDataEntry;
+import data.io.AbstractMVDataReader;
+
+/**
+ * Stream-oriented reader from a particular CSV file.
+ * Always returns lines/items sorted by lexicographical ascending key.
+ *
+ * @author lpouzenc
+ */
+public class CSVDataReader extends AbstractMVDataReader {
+
+ public static final String CSV_DEMO =
+ //"key,attr,values\n" +
+ "line3,hello,all;the;others\n" +
+ "line1,from,csv1;csv1bis\n" +
+ "line2,hello,all;the;world\n" +
+ "line1,attr2,csv1\n" +
+ ",,\n";
+
+ public static final CSVFormat DEFAULT_CSV_FORMAT = CSVFormat.EXCEL
+ .withHeader("key","attr","values")
+ .withIgnoreSurroundingSpaces(true);
+
+ private final CSVFormat format;
+ private final Reader dataSourceStream;
+
+ private transient MVDataEntry nextEntry;
+ private transient CSVRecord nextCSVRecord;
+ private transient Iterator<CSVRecord> csvIt;
+
+
+ /**
+ * Constructs a CSVDataReader object for parsing a CSV input given via dataSourceStream.
+ * @param dataSourceName A short string representing this reader (for logging)
+ * @param dataSourceStream A java.io.Reader from which read the actual CSV data, typically a FileReader
+ * @param alreadySorted If false, memory cost is around 3 times the CSV file size !
+ * @param format Specify the exact format used to encode the CSV file (separators, escaping...)
+ * @throws IOException
+ */
+ public CSVDataReader(String dataSourceName, Reader dataSourceStream, boolean alreadySorted, CSVFormat format) throws IOException {
+ this.dataSourceName = dataSourceName;
+ this.format = format;
+
+ if ( alreadySorted ) {
+ this.dataSourceStream = dataSourceStream;
+ } else {
+ BufferedReader bufReader;
+ if ( dataSourceStream instanceof BufferedReader ) {
+ bufReader = (BufferedReader) dataSourceStream;
+ } else {
+ bufReader = new BufferedReader(dataSourceStream);
+ }
+ this.dataSourceStream = readAndSortLines(bufReader);
+ }
+ }
+
+ /**
+ * Constructs a CSVDataReader object with default CSV format (for CSVParser).
+ * @param dataSourceName A short string representing this reader (for logging)
+ * @param dataSourceStream A java.io.Reader from which read the actual CSV data, typically a FileReader
+ * @param alreadySorted If false, memory cost is around 3 times the CSV file size !
+ * @throws IOException
+ */
+ public CSVDataReader(String dataSourceName, Reader dataSourceStream, boolean alreadySorted) throws IOException {
+ this(dataSourceName, dataSourceStream, alreadySorted, DEFAULT_CSV_FORMAT);
+ }
+
+ /**
+ * {@inheritDoc}
+ * Note : multiple iterators on the same instance are not supported
+ */
+ @Override
+ public Iterator<MVDataEntry> iterator() {
+ // When a new iterator is requested, everything should be reset
+ CSVParser parser;
+ try {
+ dataSourceStream.reset();
+ parser = new CSVParser(dataSourceStream, format);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ csvIt = parser.iterator();
+ nextCSVRecord = null;
+ nextEntry = null;
+ return this;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean hasNext() {
+ if ( nextEntry == null ) {
+ lookAhead();
+ }
+ return ( nextEntry != null );
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public MVDataEntry next() {
+ if ( !hasNext() ) {
+ throw new NoSuchElementException();
+ }
+ // Pop the lookahead record
+ MVDataEntry res = nextEntry;
+ nextEntry=null;
+ // And return it
+ return res;
+ }
+
+ /**
+ * In-memory File sorting, return as a single String
+ * @param reader
+ * @return
+ * @throws IOException
+ */
+ private Reader readAndSortLines(BufferedReader bufReader) throws IOException {
+ // Put all the CSV in memory, in a SortedSet
+ SortedSet<String> lineSet = new TreeSet<String>();
+ String inputLine;
+ int totalCSVSize=0;
+ while ((inputLine = bufReader.readLine()) != null) {
+ lineSet.add(inputLine);
+ totalCSVSize += inputLine.length() + 1;
+ }
+ bufReader.close(); // Closes also dataSourceStream
+
+ // Put all sorted lines in a String
+ StringBuilder allLines = new StringBuilder(totalCSVSize);
+ for ( String line: lineSet) {
+ allLines.append(line + "\n");
+ }
+ lineSet = null; // Could help the GC if the input file is huge
+
+ // Build a Java Reader from that String
+ return new StringReader(allLines.toString());
+ }
+
+ /**
+ * A MVDataEntry could be represented on many CSV lines.
+ * The key is repeated, the attr could change, the values should change (for given key/attr pair)
+ */
+ private void lookAhead() {
+ MVDataEntry currEntry = null;
+
+ boolean abort=(nextCSVRecord==null && !csvIt.hasNext()); // Nothing to crunch
+ boolean done=(nextEntry!=null); // Already looked ahead
+ while (!abort && !done) {
+ // Try to get a valid CSVRecord
+ if ( nextCSVRecord == null ) {
+ nextCSVRecord = nextValidCSVRecord();
+ }
+ // If no more CSV data
+ if ( nextCSVRecord == null ) {
+ // Maybe we have a remaining entry to return
+ if ( currEntry != null ) {
+ done=true; continue;
+ } else {
+ abort=true; continue;
+ }
+ }
+
+ // Now we have a valid CSV line to put in a MVDataEntry
+ String newKey = nextCSVRecord.get("key");
+
+
+ // If no MVDataEntry yet, it's time to create it (we have data to put into)
+ if ( currEntry == null ) {
+ currEntry = new MVDataEntry(newKey);
+ }
+ // If CSV line key matches MVDataEntry key, appends attr/values on it
+ // XXX Tricky code : following condition is always true if the previous one is true
+ if ( currEntry.getKey().equals(newKey) ) {
+ currEntry.splitAndPut(nextCSVRecord.get("attr"), nextCSVRecord.get("values"), ";");
+ nextCSVRecord = null; // Record consumed
+ } else {
+ // Keys are different, we are done (and we have remaining CSV data in nextCSVRecord)
+ done=true; continue;
+ }
+ }
+
+ nextEntry = done?currEntry:null;
+ }
+
+ /**
+ * Seek for the next valid record in the CSV file
+ * @return the next valid CSVRecord
+ */
+ private CSVRecord nextValidCSVRecord() {
+ CSVRecord res = null;
+ boolean abort = !csvIt.hasNext();
+ boolean done = false;
+ while (!abort && !done) {
+ // Try to read a CSV line
+ res = (csvIt.hasNext())?csvIt.next():null;
+
+ // Break if nothing readable
+ if ( res == null ) {
+ abort=true; continue;
+ }
+
+ // Skip invalid and empty lines
+ String key = res.get("key");
+ if ( key != null && ! key.isEmpty() ) {
+ done=true; continue;
+ }
+ }
+
+ return done?res:null;
+ }
+}
diff --git a/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java b/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java
new file mode 100644
index 0000000..3f6497b
--- /dev/null
+++ b/src/connectors/src/data/io/ldap/LDAPConnectionWrapper.java
@@ -0,0 +1,112 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io.ldap;
+
+import java.io.Closeable;
+import java.io.IOException;
+
+import com.unboundid.ldap.sdk.BindResult;
+import com.unboundid.ldap.sdk.LDAPConnection;
+import com.unboundid.ldap.sdk.LDAPConnectionOptions;
+import com.unboundid.ldap.sdk.LDAPException;
+import com.unboundid.ldap.sdk.ResultCode;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public class LDAPConnectionWrapper implements Closeable {
+
+ private final LDAPConnection conn;
+
+ /**
+ * TODO javadoc
+ * @param host
+ * @param port
+ * @param bindDN
+ * @param password
+ */
+ public LDAPConnectionWrapper(String host, int port, String bindDN, String password) {
+ LDAPConnectionOptions options = new LDAPConnectionOptions();
+ options.setAbandonOnTimeout(true);
+ options.setAllowConcurrentSocketFactoryUse(true);
+ options.setAutoReconnect(true);
+ options.setCaptureConnectStackTrace(true);
+ options.setConnectTimeoutMillis(2000); // 2 seconds
+ options.setResponseTimeoutMillis(5000); // 5 seconds
+ options.setUseSynchronousMode(false);
+
+ BindResult bindResult=null;
+ try {
+ conn = new LDAPConnection(options, host, port);
+ bindResult = conn.bind(bindDN, password);
+ }
+ catch (LDAPException e) {
+ throw new RuntimeException(e);
+ }
+
+ ResultCode resultCode = bindResult.getResultCode();
+ if ( resultCode != ResultCode.SUCCESS ) {
+ throw new RuntimeException("LDAP Bind failed : " + resultCode);
+ }
+ }
+
+ /**
+ * Builds a new reader against current connection and a LDAP baseDN.
+ *
+ * @param dataSourceName Short name of this data source (for logging)
+ * @param baseDN Search base DN (will return childs of this DN)
+ * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner
+ * @param lookAheadAmount Grab this amount of entries at once (in memory-sorted, 128 could be great)
+ * @return A new reader ready to iterate on search results
+ */
+ public LDAPFlatDataReader newFlatReader(String dataSourceName, String baseDN, String keyAttr, int lookAheadAmount) {
+ try {
+ return new LDAPFlatDataReader(dataSourceName, conn, baseDN, keyAttr, lookAheadAmount);
+ } catch (LDAPException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Builds a new writer that could insert/update/delete entries on a particular LDAP connection and baseDN.
+ *
+ * @param baseDN Search base DN (will return childs of this DN)
+ * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner
+ * @return A new writter limited on a particular baseDN
+ */
+ public LDAPFlatDataWriter newFlatWriter(String baseDN, String keyAttr) {
+ try {
+ return new LDAPFlatDataWriter(conn, baseDN, keyAttr);
+ } catch (LDAPException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Close the current ldap connection.
+ */
+ @Override
+ public void close() throws IOException {
+ this.conn.close();
+ }
+}
diff --git a/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java b/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java
new file mode 100644
index 0000000..2cc79a8
--- /dev/null
+++ b/src/connectors/src/data/io/ldap/LDAPFlatDataReader.java
@@ -0,0 +1,178 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io.ldap;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.SortedMap;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import com.unboundid.ldap.sdk.Attribute;
+import com.unboundid.ldap.sdk.Filter;
+import com.unboundid.ldap.sdk.LDAPConnection;
+import com.unboundid.ldap.sdk.LDAPException;
+import com.unboundid.ldap.sdk.SearchRequest;
+import com.unboundid.ldap.sdk.SearchResult;
+import com.unboundid.ldap.sdk.SearchResultEntry;
+import com.unboundid.ldap.sdk.SearchResultListener;
+import com.unboundid.ldap.sdk.SearchResultReference;
+import com.unboundid.ldap.sdk.SearchScope;
+
+import data.MVDataEntry;
+import data.io.AbstractMVDataReader;
+
+/**
+ * Stream-oriented reader from a particular LDAP connection
+ * Always returns lines/items sorted by lexicographical ascending key
+ * Consistent even if there is a Writer on same LDAP connection (useful for sync)
+ *
+ * @author lpouzenc
+ */
+public class LDAPFlatDataReader extends AbstractMVDataReader {
+
+ private final LDAPConnection conn;
+ private final String baseDN;
+ private final String keyAttr;
+ private final int lookAheadAmount;
+ private final SortedSet<String> keys;
+
+ private transient Iterator<String> keysItCached;
+ private transient Iterator<String> keysItConsumed;
+ private transient SortedMap<String, MVDataEntry> entries;
+
+ // Listener to feed LDAP search result in SortedMap without instantiating a big fat SearchResult
+ private final SearchResultListener keysReqListener = new SearchResultListener() {
+ private static final long serialVersionUID = 3364745402521913458L;
+
+ @Override
+ public void searchEntryReturned(SearchResultEntry searchEntry) {
+ keys.add(searchEntry.getAttributeValue(keyAttr));
+ }
+
+ @Override
+ public void searchReferenceReturned(SearchResultReference searchReference) {
+ throw new RuntimeException("Unsupported : search request for all '" + keyAttr + "' has returned at least one reference (excepected : an entry)");
+ }
+ };
+
+ /**
+ * Construct a new reader that wrap a particular LDAP search on a given connection
+ * @param dataSourceName Short name of this data source (for logging)
+ * @param conn Already initialized LDAP connection where run the search
+ * @param baseDN Search base DN (will return childs of this DN)
+ * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner
+ * @param lookAheadAmount Grab this amount of entries at once (in memory-sorted, 128 could be great)
+ * @throws LDAPException
+ */
+ public LDAPFlatDataReader(String dataSourceName, LDAPConnection conn, String baseDN, String keyAttr, int lookAheadAmount) throws LDAPException {
+ this.dataSourceName = dataSourceName;
+ this.conn = conn;
+ this.baseDN = baseDN;
+ this.keyAttr = keyAttr;
+ this.lookAheadAmount = lookAheadAmount;
+
+ // Grab all the entries' keys from LDAP connection and put them in this.keys
+ this.keys = new TreeSet<String>();
+ SearchRequest keysReq = new SearchRequest(keysReqListener, baseDN, SearchScope.ONE, Filter.create("(objectClass=*)"), keyAttr);
+ conn.search(keysReq);
+ }
+
+ /**
+ * {@inheritDoc}
+ * Note : multiple iterators on the same instance are not supported
+ */
+ @Override
+ public Iterator<MVDataEntry> iterator() {
+ // Reset the search (it uses two different iterators on the same set)
+ keysItCached = keys.iterator();
+ keysItConsumed = keys.iterator();
+ entries = new TreeMap<String, MVDataEntry>();
+
+ return this;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean hasNext() {
+ return (keysItConsumed==null)?false:keysItConsumed.hasNext();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public MVDataEntry next() {
+ String wantedKey = keysItConsumed.next();
+
+ // Feed the lookAhead buffer if it is empty (and there is more elements to grab)
+ if ( entries.isEmpty() && keysItCached.hasNext() ) {
+ lookAhead(lookAheadAmount);
+ }
+
+ //FIXME : it is possible to have inconsistency between "entries" content and keysIt* values if some entry is deleted since we have read all the keys
+
+ // Pop an entry from the lookAhead buffer
+ MVDataEntry wantedEntry = entries.remove(wantedKey);
+ if ( wantedEntry == null ) {
+ throw new NoSuchElementException();
+ }
+
+ return wantedEntry;
+ }
+
+ /**
+ * Performs look-ahead of amount entries, using the next sorted keys previously queried.
+ * @param amount
+ */
+ private void lookAhead(int amount) {
+ if ( amount < 1 ) {
+ throw new IllegalArgumentException("LookAhead amount has to be >= 1");
+ }
+ try {
+ // Build a search that matches "amount" next entries
+ Filter filter = Filter.createEqualityFilter(keyAttr, keysItCached.next());
+ for (int i=0; ( i < amount-1 ) && keysItCached.hasNext(); i++) {
+ filter = Filter.createORFilter(filter, Filter.createEqualityFilter(keyAttr, keysItCached.next()));
+ }
+ SearchRequest searchRequest = new SearchRequest(baseDN, SearchScope.ONE, filter, "*");
+
+ // XXX Could use a second listener, as for the keys
+ // Get all this entries in memory, convert them in MVDataEntry beans and store them in a SortedMap
+ SearchResult search = conn.search(searchRequest);
+
+ for (SearchResultEntry ldapEntry: search.getSearchEntries()) {
+ String key = ldapEntry.getAttributeValue(keyAttr);
+ MVDataEntry mvEntry = new MVDataEntry(key);
+
+ for ( Attribute attr : ldapEntry.getAttributes() ) {
+ mvEntry.put(attr.getName(), attr.getValues());
+ }
+ entries.put(key, mvEntry);
+ }
+ } catch (LDAPException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java b/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java
new file mode 100644
index 0000000..d1b8918
--- /dev/null
+++ b/src/connectors/src/data/io/ldap/LDAPFlatDataWriter.java
@@ -0,0 +1,198 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io.ldap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import com.unboundid.ldap.sdk.Attribute;
+import com.unboundid.ldap.sdk.DN;
+import com.unboundid.ldap.sdk.DeleteRequest;
+import com.unboundid.ldap.sdk.Entry;
+import com.unboundid.ldap.sdk.LDAPConnection;
+import com.unboundid.ldap.sdk.LDAPException;
+import com.unboundid.ldap.sdk.Modification;
+import com.unboundid.ldap.sdk.ModificationType;
+import com.unboundid.ldap.sdk.ModifyRequest;
+import com.unboundid.ldap.sdk.RDN;
+import com.unboundid.ldap.sdk.schema.EntryValidator;
+import com.unboundid.ldif.LDIFException;
+
+import data.MVDataEntry;
+import data.io.AbstractMVDataWriter;
+
+/**
+ * Stream-oriented LDAP writer from a particular LDAP Directory connection.
+ *
+ * @author lpouzenc
+ */
+public class LDAPFlatDataWriter extends AbstractMVDataWriter {
+
+ private final LDAPConnection conn;
+ private final DN baseDN;
+ private final String keyAttr;
+ private final EntryValidator validator;
+
+ /**
+ * Construct a new writer that could insert/update/delete entries on a particular LDAP connection and baseDN.
+ *
+ * @param conn Already initialized LDAP connection where run the search
+ * @param baseDN Search base DN (will return childs of this DN)
+ * @param keyAttr Attribute name that is the primary key of the entry, identifying the entry in a unique manner
+ * @throws LDAPException
+ */
+ public LDAPFlatDataWriter(LDAPConnection conn, String baseDN, String keyAttr) throws LDAPException {
+ this.conn = conn;
+ this.baseDN = new DN(baseDN);
+ this.keyAttr = keyAttr;
+ this.validator = new EntryValidator(conn.getSchema());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void insert(MVDataEntry newEntry) throws LDAPException {
+ // Build the DN
+ DN dn = new DN(new RDN(keyAttr, newEntry.getKey()), baseDN);
+
+ // Convert storage objects
+ Collection<Attribute> attributes = new ArrayList<Attribute>();
+ for ( Map.Entry<String, String> entry : newEntry.getAllEntries() ) {
+ attributes.add(new Attribute(entry.getKey(), entry.getValue()));
+ }
+ Entry newLDAPEntry = new Entry(dn, attributes);
+
+ // Add the entry
+ if ( dryRun ) {
+ // In dry-run mode, validate the entry
+ ArrayList<String> invalidReasons = new ArrayList<String>(5);
+ boolean valid = validator.entryIsValid(newLDAPEntry, invalidReasons);
+ if ( !valid ) throw new RuntimeException(
+ "Entry validator has failed to verify this entry :\n" + newLDAPEntry.toLDIFString() +
+ "Reasons are :\n" + invalidReasons);
+ } else {
+ // In real-run mode, insert the entry
+ try {
+ conn.add(newLDAPEntry);
+ } catch (LDAPException e) {
+ throw new LDAPException(e.getResultCode(), "Error while inserting this entry :\n" + newLDAPEntry.toLDIFString(), e);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void update(MVDataEntry updatedEntry, MVDataEntry originalEntry, Set<String> attrToUpdate) throws LDAPException, LDIFException {
+ // Build the DN
+ DN dn = new DN(new RDN(keyAttr, updatedEntry.getKey()), baseDN);
+
+ // Convert storage objects
+ List<Modification> mods = new ArrayList<Modification>();
+ for ( String attr : attrToUpdate ) {
+ Set<String> originalValues = originalEntry.getValues(attr);
+ Set<String> updatedValues = updatedEntry.getValues(attr);
+
+ Modification modification = null;
+
+ if ( updatedValues.isEmpty() ) {
+ modification = new Modification(ModificationType.DELETE, attr);
+ } else {
+ String[] updatedValuesArr = updatedValues.toArray(new String[0]);
+
+ if ( originalValues.isEmpty() ) {
+ modification = new Modification(ModificationType.ADD, attr, updatedValuesArr);
+ } else {
+ modification = new Modification(ModificationType.REPLACE, attr, updatedValuesArr);
+ }
+ }
+
+ mods.add(modification);
+ }
+ ModifyRequest modReq = new ModifyRequest(dn, mods);
+
+ // Update the entry
+ if ( dryRun ) {
+ // Simulate originalEntry update
+ Collection<Attribute> attributes = new ArrayList<Attribute>();
+ for ( Map.Entry<String, String> entry : originalEntry.getAllEntries() ) {
+ attributes.add(new Attribute(entry.getKey(), entry.getValue()));
+ }
+ Entry originalLDAPEntry = new Entry(dn, attributes);
+
+ // Warning : Unboundid SDK is okay with mandatory attributes with value "" (empty string)
+ // OpenLDAP do not allow that empty strings in mandatory attributes.
+ // Empty strings are discarded by MVDataEntry.put() for now.
+ Entry modifiedLDAPEntry;
+ try {
+ modifiedLDAPEntry = Entry.applyModifications(originalLDAPEntry, false, mods);
+ } catch (LDAPException originalException) {
+ throw new RuntimeException("Entry update simulation has failed while running applyModifications()\n"
+ + "original entry : " + originalEntry + "\n"
+ + "wanted updated entry : " + updatedEntry + "\n"
+ + "modification request : " + modReq,
+ originalException);
+ }
+ ArrayList<String> invalidReasons = new ArrayList<String>(5);
+ boolean valid = validator.entryIsValid(modifiedLDAPEntry, invalidReasons);
+ if ( !valid ) throw new RuntimeException("Entry update simulation has failed while checking entryIsValid()\n"
+ + "modified entry : " + modifiedLDAPEntry.toLDIFString() + "\n"
+ + "reasons :" + invalidReasons);
+ } else {
+ // In real-run mode, update the entry
+ try {
+ conn.modify(modReq);
+ } catch (LDAPException originalException) {
+ throw new LDAPException(originalException.getResultCode(),
+ "Error while updating this entry :\n" + modReq.toLDIFString(),
+ originalException);
+ }
+ }
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public void delete(MVDataEntry existingEntry) throws LDAPException {
+ // Build the DN
+ DN dn = new DN(new RDN(keyAttr, existingEntry.getKey()), baseDN);
+
+ // Delete the entry
+ try {
+ if ( dryRun ) {
+ //XXX : try to verify the entry existence in dry-run mode ?
+ } else {
+ conn.delete(new DeleteRequest(dn));
+ }
+ } catch (LDAPException originalException) {
+ throw new LDAPException(originalException.getResultCode(),
+ "Error while deleting this dn : " + dn.toString(),
+ originalException);
+ }
+ }
+
+}
diff --git a/src/connectors/src/data/io/sql/SQLConnectionWrapper.java b/src/connectors/src/data/io/sql/SQLConnectionWrapper.java
new file mode 100644
index 0000000..2bab2c8
--- /dev/null
+++ b/src/connectors/src/data/io/sql/SQLConnectionWrapper.java
@@ -0,0 +1,136 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io.sql;
+
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.Driver;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+import data.io.MVDataReader;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public class SQLConnectionWrapper implements Closeable {
+
+ /**
+ * Enumeration of supported DBMS. Each use a particular JDBC driver.
+ */
+ public enum DBMSType { oracle, mysql/*, derby*/ }
+
+ private final Connection conn;
+
+ /**
+ * TODO javadoc
+ * @param dbms
+ * @param host
+ * @param port
+ * @param ress
+ * @param user
+ * @param pass
+ * @param db
+ */
+ public SQLConnectionWrapper(DBMSType dbms, String host, int port, String ress, String user, String pass, String db) {
+
+ String driverClassName=null;
+ String url;
+
+ switch ( dbms ) {
+ case oracle:
+ driverClassName="oracle.jdbc.driver.OracleDriver";
+ url="jdbc:oracle:thin:@" + host + ":" + port + ":" + ress + "/" + db;
+ break;
+ case mysql:
+ driverClassName="com.mysql.jdbc.Driver";
+ url="jdbc:mysql://" + host + ":" + port + "/" + db;
+ break;
+ /* Could be useful with JUnit tests
+ case derby:
+ driverClassName="org.apache.derby.jdbc.EmbeddedDriver";
+ url="jdbc:derby:" + db;
+ break;
+ */
+ default:
+ throw new IllegalArgumentException("Unsupported DBMSType : " + dbms);
+ }
+
+ try {
+ @SuppressWarnings("unchecked")
+ Class<? extends Driver> clazz = (Class<? extends Driver>) Class.forName(driverClassName);
+ DriverManager.registerDriver(clazz.newInstance());
+ } catch (Exception e) {
+ throw new RuntimeException("Can't load or register JDBC driver for " + dbms + " (" + driverClassName + ")", e);
+ }
+
+ try {
+ conn = DriverManager.getConnection(url, user, pass);
+ } catch (SQLException e) {
+ throw new RuntimeException("Can't establish database connection (" + url + ")");
+ }
+ }
+
+ /**
+ * Builds a new reader against current connection and a File containing a SELECT statement.
+ * @param name
+ * @param queryFile
+ * @return
+ * @throws IOException
+ */
+ public MVDataReader newReader(String name, File queryFile) throws IOException {
+ return new SQLRelDataReader(name, conn, queryFile);
+ }
+
+ /**
+ * Builds a new reader against current connection and a String containing a SELECT statement.
+ * @param name
+ * @param query
+ * @return
+ * @throws IOException
+ */
+ public MVDataReader newReader(String name, String query) {
+ return new SQLRelDataReader(name, conn, query);
+ }
+
+ /**
+ * Close the current database connection.
+ */
+ @Override
+ public void close() throws IOException {
+ try {
+ conn.close();
+ } catch (SQLException e) {
+ throw new IOException("Exception occured while trying to close the SQL connection", e);
+ }
+ }
+
+ /**
+ * @return the current database connection (useful for JUnit tests)
+ */
+ public Connection getConn() {
+ return conn;
+ }
+}
diff --git a/src/connectors/src/data/io/sql/SQLRelDataReader.java b/src/connectors/src/data/io/sql/SQLRelDataReader.java
new file mode 100644
index 0000000..b6355e9
--- /dev/null
+++ b/src/connectors/src/data/io/sql/SQLRelDataReader.java
@@ -0,0 +1,173 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io.sql;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Iterator;
+
+import data.MVDataEntry;
+import data.io.AbstractMVDataReader;
+
+/**
+ * Stream-oriented reader from a particular RDBMS source.
+ *
+ * @author lpouzenc
+ */
+public class SQLRelDataReader extends AbstractMVDataReader {
+
+ private final Connection conn;
+ private final String request;
+
+ private transient String columnNames[];
+ private transient ResultSet rs;
+ private transient boolean didNext;
+ private transient boolean hasNext;
+
+ /**
+ * Build a new reader from an existing connection and a File containing a SELECT statement.
+ * @param dataSourceName A short string representing this reader (for logging)
+ * @param conn A pre-established SQL data connection
+ * @param queryFile An SQL file containing an SQL SELECT statement
+ * @throws IOException
+ */
+ public SQLRelDataReader(String dataSourceName, Connection conn, File queryFile) throws IOException {
+ this.dataSourceName = dataSourceName;
+ this.conn = conn;
+ this.request = readEntireFile(queryFile);
+ }
+
+ /**
+ * Build a new reader from an existing connection and a String containing a SELECT statement.
+ * @param dataSourceName A short string representing this reader (for logging)
+ * @param conn A pre-established SQL data connection
+ * @param query A String containing an SQL SELECT statement
+ * @throws IOException
+ */
+ public SQLRelDataReader(String dataSourceName, Connection conn, String query) {
+ this.dataSourceName = dataSourceName;
+ this.conn = conn;
+ this.request = query;
+ }
+
+ /**
+ * {@inheritDoc}
+ * Note : multiple iterators on the same instance are not supported
+ */
+ @Override
+ public Iterator<MVDataEntry> iterator() {
+ try {
+ // Reset iterator-related attributes
+ hasNext = false;
+ didNext = false;
+
+ // Close and free any previous request result
+ if ( rs != null ) {
+ rs.close();
+ }
+ // (Re-)Execute the SQL request
+ Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
+ rs = stmt.executeQuery(request);
+
+ // Get the column names
+ ResultSetMetaData rsmd = rs.getMetaData();
+ columnNames = new String[rsmd.getColumnCount()];
+ for (int i = 0; i < columnNames.length ; i++) {
+ // Java SQL : all indices starts at 1 (it sucks !)
+ columnNames[i] = rsmd.getColumnName(i+1);
+ }
+ } catch (SQLException e) {
+ throw new RuntimeException("Could not execute query : " + e.getMessage() + "\n" + request );
+ }
+
+ return this;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean hasNext() {
+ // java.sql.ResultSet don't implement Iterable interface at all
+ // It's next() don't return anything except hasNext() result but it moves the cursor !
+ if (!didNext) {
+ try {
+ hasNext = rs.next();
+ } catch (SQLException e) {
+ throw new RuntimeException(e);
+ }
+ didNext = true;
+ }
+ return hasNext;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public MVDataEntry next() {
+ MVDataEntry result = null;
+ try {
+ if (!didNext) {
+ rs.next();
+ }
+ didNext = false;
+ //TODO Instead of always use the first col, user could choose a specific columnName like in LDAP
+ String key = rs.getString(1);
+ result = new MVDataEntry(key);
+ for (int i = 0; i < columnNames.length ; i++) {
+ // Java SQL : all indices starts at 1 (it sucks !)
+ result.splitAndPut(columnNames[i], rs.getString(i+1), ";"); // TODO regex should be an option
+ }
+
+ } catch (SQLException e) {
+ throw new RuntimeException("Exception while reading next line in SQL resultset", e);
+ }
+
+ return result;
+ }
+
+ /**
+ * Helper function to load and entire file as a String.
+ * @param file
+ * @return
+ * @throws IOException
+ */
+ private static String readEntireFile(File file) throws IOException {
+ FileReader input = new FileReader(file);
+ StringBuilder contents = new StringBuilder();
+ char[] buffer = new char[4096];
+ int read = 0;
+ do {
+ contents.append(buffer, 0, read);
+ read = input.read(buffer);
+ } while (read >= 0);
+ input.close();
+
+ return contents.toString();
+ }
+}
diff --git a/src/core/.classpath b/src/core/.classpath
new file mode 100644
index 0000000..f7de406
--- /dev/null
+++ b/src/core/.classpath
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+ <classpathentry kind="src" path="JUTests"/>
+ <classpathentry kind="lib" path="lib/guava-16.0.1.jar"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
diff --git a/src/core/.project b/src/core/.project
new file mode 100644
index 0000000..acda864
--- /dev/null
+++ b/src/core/.project
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>SSSync_Core</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
diff --git a/src/core/.settings/org.eclipse.jdt.core.prefs b/src/core/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..8000cd6
--- /dev/null
+++ b/src/core/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,11 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.6
diff --git a/src/core/JUTests/data/MVDataEntryTest.java b/src/core/JUTests/data/MVDataEntryTest.java
new file mode 100644
index 0000000..19ccb46
--- /dev/null
+++ b/src/core/JUTests/data/MVDataEntryTest.java
@@ -0,0 +1,93 @@
+package data;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+import com.google.common.collect.HashMultimap;
+
+public class MVDataEntryTest {
+
+ @Test
+ public void testMVDataEntryStringIntInt() {
+ String expected = "{key=line1, attrValPairs={k4=[v4], k1=[v1b, v1a, v1c], k2=[v1c]}}";
+
+ MVDataEntry e1 = new MVDataEntry("line1", 1, 1);
+ HashMultimap<String, String> e1v = e1.getAttrValPairs();
+ e1v.put("k1", "v1a");
+ e1v.put("k1", "v1b");
+ e1v.put("k1", "v1b"); // Twice, should disappear silently
+ e1v.put("k1", "v1c");
+ e1v.put("k2", "v1c");
+ e1v.put("k4", "v4");
+
+ assertEquals(expected, e1.toString());
+
+ }
+
+ @Test
+ public void testMerge() {
+ // Test data
+ MVDataEntry e1 = new MVDataEntry("10");
+ HashMultimap<String, String> e1v = e1.getAttrValPairs();
+ e1v.put("k1", "v1a");
+ e1v.put("k1", "v1b");
+ e1v.put("k1", "v1c");
+ e1v.put("k2", "v2");
+ e1v.put("k4", "v4");
+
+ MVDataEntry e2 = new MVDataEntry("2");
+ HashMultimap<String, String> e2v = e2.getAttrValPairs();
+ e2v.put("k2", "v2");
+ e2v.put("k1", "v1b");
+ e2v.put("k3", "v3");
+
+ MVDataEntry r1 = new MVDataEntry(e1);
+ r1.mergeValues(true, e2);
+ assertNotSame(r1, e1);
+ String expected1 = "{key=10, attrValPairs={k3=[v3], k4=[v4], k1=[v1b, v1a, v1c], k2=[v2]}}";
+ assertEquals(expected1, r1.toString());
+
+ MVDataEntry r2 = new MVDataEntry(e2);
+ r2.mergeValues(true, e1);
+ assertNotSame(r2, e2);
+ String expected2 = "{key=2, attrValPairs={k3=[v3], k4=[v4], k1=[v1b, v1a, v1c], k2=[v2]}}";
+ assertEquals(expected2, r2.toString());
+
+ MVDataEntry r3 = new MVDataEntry(e1);
+ r3.mergeValues(false, e2);
+ assertNotSame(r3, e1);
+ String expected3 = "{key=10, attrValPairs={k3=[v3], k4=[v4], k1=[v1b], k2=[v2]}}";
+ //System.out.println(expected3);
+ //System.out.println(r3.toString());
+ assertEquals(expected3, r3.toString());
+
+ MVDataEntry r4 = new MVDataEntry(e2);
+ r4.mergeValues(false, e1);
+ assertNotSame(r4, e1);
+ String expected4 = "{key=2, attrValPairs={k3=[v3], k4=[v4], k1=[v1b, v1a, v1c], k2=[v2]}}";
+ assertEquals(expected4, r4.toString());
+
+ assertTrue(!r2.equals(r3));
+ assertEquals(r2,r4);
+ }
+
+ @Test
+ public void testSplitAndPut() {
+ MVDataEntry r1 = new MVDataEntry("10");
+ r1.splitAndPut("k1", "v1a;v1b;v1c", ";");
+ r1.put("k2", "v2", null); // splitAndPut does not support null regex anymore, use put()
+ r1.splitAndPut("k4", "v4", "^$");
+
+ MVDataEntry expected1 = new MVDataEntry("10");
+ HashMultimap<String, String> expected1v = expected1.getAttrValPairs();
+ expected1v.put("k1", "v1a");
+ expected1v.put("k1", "v1b");
+ expected1v.put("k1", "v1c");
+ expected1v.put("k2", "v2");
+ expected1v.put("k4", "v4");
+
+ assertEquals(r1,expected1);
+ }
+
+}
diff --git a/src/core/JUTests/data/io/filters/MVDataCombinerTest.java b/src/core/JUTests/data/io/filters/MVDataCombinerTest.java
new file mode 100644
index 0000000..5d32dd8
--- /dev/null
+++ b/src/core/JUTests/data/io/filters/MVDataCombinerTest.java
@@ -0,0 +1,148 @@
+package data.io.filters;
+
+import static org.junit.Assert.*;
+
+import java.util.Iterator;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import data.MVDataEntry;
+import data.filters.MVDataCombiner;
+import data.filters.MVDataCombiner.MVDataCombineMode;
+import data.io.MVDataReader;
+import data.io.stub.StubDataReader;
+
+public class MVDataCombinerTest {
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ @Test
+ public void testOutOfOrderCase() {
+ // Test Data
+ MVDataEntry e10 = new MVDataEntry("line2");
+ e10.getAttrValPairs().put("merge", "e10");
+ MVDataEntry e11 = new MVDataEntry("line1");
+ e11.getAttrValPairs().put("merge", "e11");
+
+ MVDataEntry e21 = new MVDataEntry("line2");
+ e21.getAttrValPairs().put("merge", "e21");
+
+ MVDataEntry[][] fakeEntries = new MVDataEntry[][] {
+ new MVDataEntry[] { e10, e11 },
+ new MVDataEntry[] { e21 },
+ };
+
+ MVDataCombineMode mergeModes[] = new MVDataCombineMode[]{
+ MVDataCombineMode.PRIMARY_SOURCE,
+ MVDataCombineMode.MERGE_APPEND,
+ };
+
+ // Expected results
+ MVDataEntry line1 = new MVDataEntry(e10);
+ line1.mergeValues(true, e21);
+
+ MVDataEntry expected[] = new MVDataEntry[] {
+ line1,
+ null /* Should throw UnsupportedOperationException() before comparing */
+ };
+
+ // Test run
+ exception.expect(UnsupportedOperationException.class);
+ doCombineTest(expected, fakeEntries, mergeModes);
+ }
+
+
+ @Test
+ public void testGeneralCase() {
+
+ // Test Data
+ MVDataEntry e10 = new MVDataEntry("line3");
+ e10.getAttrValPairs().put("from1", "e10");
+ e10.getAttrValPairs().put("merge", "e10");
+ MVDataEntry e11 = new MVDataEntry("line4");
+ e11.getAttrValPairs().put("from1", "e11");
+ e11.getAttrValPairs().put("merge", "e11");
+
+ MVDataEntry e20 = new MVDataEntry("line1");
+ e20.getAttrValPairs().put("from2", "e20");
+ e20.getAttrValPairs().put("merge", "e20");
+ MVDataEntry e21 = new MVDataEntry("line2");
+ e21.getAttrValPairs().put("from2", "e21");
+ e21.getAttrValPairs().put("merge", "e21");
+ MVDataEntry e22 = new MVDataEntry("line3");
+ e22.getAttrValPairs().put("from2", "e22");
+ e22.getAttrValPairs().put("merge", "e22");
+
+ MVDataEntry e30 = new MVDataEntry("line2");
+ e30.getAttrValPairs().put("from3", "e30");
+ e30.getAttrValPairs().put("merge", "e30");
+
+
+ MVDataEntry[][] fakeEntries = new MVDataEntry[][] {
+ new MVDataEntry[] { e10, e11 },
+ new MVDataEntry[] { e20, e21, e22 },
+ new MVDataEntry[] { e30 },
+ };
+
+ MVDataCombineMode mergeModes[] = new MVDataCombineMode[]{
+ MVDataCombineMode.PRIMARY_SOURCE,
+ MVDataCombineMode.MERGE_REPLACE,
+ MVDataCombineMode.MERGE_APPEND,
+ };
+
+ // Expected results
+ MVDataEntry line1 = new MVDataEntry(e20);
+
+ MVDataEntry line2 = new MVDataEntry(e21);
+ line2.mergeValues(true, e30);
+
+ MVDataEntry line3 = new MVDataEntry(e10);
+ line3.mergeValues(false, e22);
+
+ MVDataEntry line4 = new MVDataEntry(e11);
+
+ MVDataEntry expected[] = new MVDataEntry[] {
+ line1,line2,line3,line4
+ };
+
+ // Test run
+ doCombineTest(expected, fakeEntries, mergeModes);
+ }
+
+ // TODO : test all Combine modes
+
+ /**
+ * Helper function to factorise Combiner tests.
+ * @param expected
+ * @param fakeEntries
+ * @param mergeModes
+ */
+ public void doCombineTest(MVDataEntry expected[], MVDataEntry[][] fakeEntries, MVDataCombineMode mergeModes[]) {
+ // Test init
+ MVDataReader readers[] = new MVDataReader[fakeEntries.length];
+ for (int i = 0; i < fakeEntries.length; i++) {
+ readers[i] = new StubDataReader("fakeReader"+i,fakeEntries[i]);
+ }
+
+ MVDataCombiner combiner = new MVDataCombiner("combiner", readers, mergeModes);
+
+ // Test twice to check if asking a new iterator "rewinds" correctly
+ for (int i=0;i<2;i++) {
+ //System.out.println("Loop " + (i+1));
+
+ Iterator<MVDataEntry> combinerIt = combiner.iterator();
+ for (int j = 0; j < expected.length; j++) {
+ assertTrue(combinerIt.hasNext());
+ MVDataEntry item = combinerIt.next();
+ //System.out.println(expected[i]);
+ //System.out.println(item);
+ //System.out.println();
+ assertEquals(expected[j], item);
+ }
+ assertFalse(combinerIt.hasNext());
+ }
+ }
+}
diff --git a/src/core/build.xml b/src/core/build.xml
new file mode 100644
index 0000000..e46c220
--- /dev/null
+++ b/src/core/build.xml
@@ -0,0 +1,78 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- WARNING: Eclipse auto-generated file.
+ Any modifications will be overwritten.
+ To include a user specific buildfile here, simply create one in the same
+ directory with the processing instruction <?eclipse.ant.import?>
+ as the first entry and export the buildfile again. -->
+<project basedir="." default="build" name="SSSync_Core">
+ <property environment="env"/>
+ <property name="SSSync_Connectors.location" value="../connectors"/>
+ <property name="SSSync_Main.location" value="../main"/>
+ <property name="ECLIPSE_HOME" value="../../../../../../usr/lib/eclipse"/>
+ <property name="debuglevel" value="source,lines,vars"/>
+ <property name="target" value="1.6"/>
+ <property name="source" value="1.6"/>
+ <path id="JUnit 4.libraryclasspath">
+ <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.junit_4.8.2.dist/junit.jar"/>
+ <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.hamcrest.core_1.1.0.jar"/>
+ </path>
+ <path id="SSSync_Core.classpath">
+ <pathelement location="bin"/>
+ <pathelement location="lib/guava-16.0.1.jar"/>
+ <path refid="JUnit 4.libraryclasspath"/>
+ </path>
+ <target name="init">
+ <mkdir dir="bin"/>
+ <copy includeemptydirs="false" todir="bin">
+ <fileset dir="src">
+ <exclude name="**/*.java"/>
+ </fileset>
+ </copy>
+ <copy includeemptydirs="false" todir="bin">
+ <fileset dir="JUTests">
+ <exclude name="**/*.java"/>
+ </fileset>
+ </copy>
+ </target>
+ <target name="clean">
+ <delete dir="bin"/>
+ </target>
+ <target depends="clean" name="cleanall"/>
+ <target depends="build-subprojects,build-project" name="build"/>
+ <target name="build-subprojects"/>
+ <target depends="init" name="build-project">
+ <echo message="${ant.project.name}: ${ant.file}"/>
+ <javac debug="true" debuglevel="${debuglevel}" destdir="bin" includeantruntime="false" source="${source}" target="${target}">
+ <src path="src"/>
+ <src path="JUTests"/>
+ <classpath refid="SSSync_Core.classpath"/>
+ </javac>
+ </target>
+ <target description="Build all projects which reference this project. Useful to propagate changes." name="build-refprojects">
+ <ant antfile="build.xml" dir="${SSSync_Connectors.location}" inheritAll="false" target="clean"/>
+ <ant antfile="build.xml" dir="${SSSync_Connectors.location}" inheritAll="false" target="build">
+ <propertyset>
+ <propertyref name="build.compiler"/>
+ </propertyset>
+ </ant>
+ <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="clean"/>
+ <ant antfile="build.xml" dir="${SSSync_Main.location}" inheritAll="false" target="build">
+ <propertyset>
+ <propertyref name="build.compiler"/>
+ </propertyset>
+ </ant>
+ </target>
+ <target description="copy Eclipse compiler jars to ant lib directory" name="init-eclipse-compiler">
+ <copy todir="${ant.library.dir}">
+ <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/>
+ </copy>
+ <unzip dest="${ant.library.dir}">
+ <patternset includes="jdtCompilerAdapter.jar"/>
+ <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/>
+ </unzip>
+ </target>
+ <target description="compile project with Eclipse compiler" name="build-eclipse-compiler">
+ <property name="build.compiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/>
+ <antcall target="build"/>
+ </target>
+</project>
diff --git a/src/core/lib/guava-16.0.1.jar b/src/core/lib/guava-16.0.1.jar
new file mode 100644
index 0000000..2c8127d
--- /dev/null
+++ b/src/core/lib/guava-16.0.1.jar
Binary files differ
diff --git a/src/core/src/data/MVDataEntry.java b/src/core/src/data/MVDataEntry.java
new file mode 100644
index 0000000..f92a141
--- /dev/null
+++ b/src/core/src/data/MVDataEntry.java
@@ -0,0 +1,238 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data;
+
+import java.util.HashSet;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import com.google.common.collect.HashMultimap;
+
+/**
+ * Generic Multi-Valued data type. Each object store a particular entry.
+ * Semantics are like in LDAP directories : an entry = a key + a set of multi-valued attributes.
+ * Relational data like in RDMS are more constrained : columns are fixed for an entire table.
+ * Null and empty string attribute value are silently discarded.
+ *
+ * @author lpouzenc
+ */
+public class MVDataEntry implements Comparable<MVDataEntry> {
+
+ /**
+ * The key part that identify this particular entry.
+ */
+ private final String key;
+ /**
+ * The data part of this particular entry.
+ */
+ private HashMultimap<String,String> attrValPairs;
+
+ // XXX : add an HashMap for meta or constraints ?
+
+ // Constructors
+
+ /**
+ * Build a fresh empty MVDataEntry.
+ * @param key Unique key identifying this entry
+ */
+ public MVDataEntry(String key) {
+ if ( key == null ) {
+ throw new IllegalArgumentException("key must be non-null");
+ }
+ this.key = key;
+ this.attrValPairs = HashMultimap.create();
+ }
+
+ /**
+ * Build a fresh empty MVDataEntry with hints about expected attr/values count.
+ * @param key Unique key identifying this entry
+ */
+ public MVDataEntry(String key, int expectedAttribs, int expectedValuesPerAttrib) {
+ if ( key == null ) {
+ throw new IllegalArgumentException("key must be non-null");
+ }
+ this.key = key;
+ this.attrValPairs = HashMultimap.create(expectedAttribs, expectedValuesPerAttrib);
+ }
+
+ /**
+ * Deep copy of an existing MVDataEntry.
+ * @param key Unique key identifying this entry
+ */
+ public MVDataEntry(final MVDataEntry copyFrom) {
+ this.key=copyFrom.key; // String is immutable, so ref copy is okay
+ this.attrValPairs = HashMultimap.create(copyFrom.attrValPairs);
+ }
+
+ /**
+ * Proxy function to return all attribute/value pairs.
+ * One can use read a MVDataEntry without depending on non-standard HashMultimap.
+ * @return
+ */
+ public Set<Entry<String, String>> getAllEntries() {
+ return this.attrValPairs.entries();
+ }
+
+ /**
+ * Proxy function to add an attribute/value pair in attrValPairs.
+ * One can use MVDataEntry without depending on non-standard HashMultimap.
+ *
+ * @param attr
+ * @param value
+ */
+ public void put(String attr, String... values) {
+ for (String value: values) {
+ if ( value != null && !value.isEmpty() ) {
+ this.attrValPairs.put(attr, value);
+ }
+ }
+ }
+
+ /**
+ * Proxy function to get all values from a particular attribute.
+ * One can use MVDataEntry without depending on non-standard HashMultimap.
+ * @param attr
+ * @return
+ */
+ public Set<String> getValues(String attr) {
+ return this.attrValPairs.get(attr);
+ }
+
+ /**
+ * Helper function to insert multiple values from a single string.
+ *
+ * @param attr
+ * @param value
+ * @param splitRegex
+ */
+ public void splitAndPut(String attr, String value, String splitRegex) {
+ if ( value != null ) {
+ for (String v : value.split(splitRegex)) {
+ put(attr, v);
+ }
+ }
+ }
+
+ /**
+ * Helper function to return list of changed attributes.
+ * Note : this don't keep track of deleted attributes.
+ * @param original
+ * @return
+ */
+ public Set<String> getChangedAttributes(MVDataEntry original) {
+ HashSet<String> result = new HashSet<String>();
+
+ for (String attr: this.attrValPairs.keySet()) {
+ Set<String> thisValue = this.attrValPairs.get(attr);
+ Set<String> originalValue = original.attrValPairs.get(attr);
+ if ( ! thisValue.equals(originalValue) ) {
+ result.add(attr);
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * Augment this entry with attr/values from other entries.
+ * @param appendMode Select behavior on an existing attribute : append values or replace them
+ * @param entries Entries to merge with current entry
+ */
+ public void mergeValues(boolean appendMode, MVDataEntry... entries) {
+ for(MVDataEntry entry : entries) {
+ if ( ! appendMode ) {
+ for (String attr : entry.attrValPairs.keySet()) {
+ this.attrValPairs.removeAll(attr);
+ }
+ }
+ this.attrValPairs.putAll(entry.attrValPairs);
+ }
+ }
+
+ /**
+ * Check if this entry seems contains useful data.
+ * @return true if this entry seems contains useful data
+ */
+ public boolean isValid() {
+ boolean validKey=(this.key != null && this.key.length() > 0 );
+ boolean validVal=(this.attrValPairs != null && ! this.attrValPairs.isEmpty());
+
+ return (validKey && validVal);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean equals(Object obj) {
+ // Check for self-comparison (compare object references)
+ if ( this == obj ) { return true; }
+ // Check non-nullity and type
+ if ( !( obj instanceof MVDataEntry) ) { return false; }
+ // Cast safely
+ MVDataEntry other = (MVDataEntry) obj;
+ // Check all fields (known to be always non null)
+ return ( this.key.equals(other.key) && this.attrValPairs.equals(other.attrValPairs) );
+ }
+
+ /**
+ * Compares entries. Ordering of entries is the ordering of their keys.
+ * (java.lang.String default ordering : lexicographical ascending order)
+ */
+ @Override
+ public int compareTo(MVDataEntry other) {
+ return this.key.compareTo(other.key);
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public String toString() {
+ return "{key=" + key + ", attrValPairs=" + attrValPairs.toString() + "}";
+ }
+
+
+ // Boring accessors
+ /**
+ * @return the attrValPairs
+ */
+ public HashMultimap<String, String> getAttrValPairs() {
+ return attrValPairs;
+ }
+
+ /**
+ * @param attrValPairs the attrValPairs to set
+ */
+ public void setAttrValPairs(HashMultimap<String, String> attrValPairs) {
+ this.attrValPairs = attrValPairs;
+ }
+
+ /**
+ * @return the key (guaranteed to be non-null)
+ */
+ public String getKey() {
+ return key;
+ }
+
+
+
+}
diff --git a/src/core/src/data/filters/MVDataCombiner.java b/src/core/src/data/filters/MVDataCombiner.java
new file mode 100644
index 0000000..1b2eb3f
--- /dev/null
+++ b/src/core/src/data/filters/MVDataCombiner.java
@@ -0,0 +1,164 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.filters;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import data.MVDataEntry;
+import data.io.AbstractMVDataReader;
+import data.io.MVDataReader;
+
+/**
+ * Combines arbitrary number of MVData* sources while behaving same as AbstractMVDataReader.
+ * This could enable a sync implementation to merge multiple sources
+ * before sync'ing in a transparent manner.
+ * To prevent memory consumption, this assumes that all sources will be read
+ * with lexicographical ascending order on the "key" field.
+ *
+ * @author lpouzenc
+ */
+public class MVDataCombiner extends AbstractMVDataReader {
+
+ public enum MVDataCombineMode { PRIMARY_SOURCE, MERGE_APPEND, MERGE_REPLACE, OVERRIDE };
+
+ private final MVDataReader[] readers;
+ private final MVDataCombineMode[] mergeModes;
+
+ private transient Iterator<MVDataEntry>[] readerIterators;
+ private transient MVDataEntry[] lookAheadData;
+ private transient String lastKey;
+
+
+ public MVDataCombiner(String dataSourceName, MVDataReader[] readers, MVDataCombineMode mergeModes[]) {
+ if ( readers == null || mergeModes == null || (mergeModes.length != readers.length) ) {
+ throw new IllegalArgumentException("readers and mergeModes arrays should have same size");
+ }
+ if ( ! (mergeModes.length > 0) || mergeModes[0] != MVDataCombineMode.PRIMARY_SOURCE ) {
+ throw new IllegalArgumentException("MVDataCombiner first mergeModes should always be PRIMARY_SOURCE");
+ }
+
+ this.dataSourceName = dataSourceName;
+ this.readers = readers.clone();
+ this.mergeModes = mergeModes.clone();
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ @SuppressWarnings("unchecked") /* for new Iterator[...] */
+ public Iterator<MVDataEntry> iterator() {
+ // Be cautious to reset everything
+ readerIterators = new Iterator[readers.length];
+ for (int i=0; i<readers.length;i++) {
+ readerIterators[i] = readers[i].iterator();
+ }
+ lookAheadData = new MVDataEntry[readers.length];
+ lastKey = null;
+
+ return this;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean hasNext() {
+ for ( MVDataEntry line : lookAheadData ) {
+ if ( line != null ) { return true; }
+ }
+ for ( MVDataReader reader : readers ) {
+ if ( reader.hasNext() ) { return true; }
+ }
+ return false;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public MVDataEntry next() {
+
+ final String currentKey = lookAheadAll();
+
+ // Check if there was unsorted lines in source data
+ if ( lastKey != null && (currentKey.compareTo(lastKey) < 0) ) {
+ //XXX : this is checked here and in SafeDataReader (redundant), but both are optionnal...
+ throw new UnsupportedOperationException("At least one data source is out of order. " +
+ "Data sources are excepted to be read sorted by MVDataEntry key (ascending lexicogrpahical order)");
+ }
+
+ // Merge all data sources for key currentKey
+ MVDataEntry result = null;
+ for ( int i=0; i<lookAheadData.length; i++) {
+ if ( lookAheadData[i] != null && lookAheadData[i].getKey().equals(currentKey) ) {
+ if ( result == null ) {
+ result = lookAheadData[i];
+ } else {
+ //XXX : some items in LDAP could have constrains like : "not multi-valued". Force MERGE_REPLACE mode ?
+ //FIXME : honor all Combine modes
+ result.mergeValues( (mergeModes[i] == MVDataCombineMode.MERGE_APPEND ),lookAheadData[i]);
+ }
+ lookAheadData[i]=null; // "Pop" the used entry
+ }
+ }
+
+ lastKey = currentKey;
+
+ return result;
+ }
+
+ private String lookAheadAll() {
+ String minKey=null;
+
+ // Feed the look-ahead buffer (look forward by 1 value for each reader)
+ for ( int i=0; i<lookAheadData.length; i++) {
+ if ( lookAheadData[i] == null && readerIterators[i].hasNext() ) {
+ lookAheadData[i] = readerIterators[i].next();
+ }
+ }
+
+ // Find the least RelData key from look-ahead buffers
+ for (MVDataEntry entry: lookAheadData) {
+ if ( entry != null ) {
+ final String minKeyCandidate = entry.getKey();
+ if ( minKey == null || minKey.compareTo(minKeyCandidate) > 0 ) {
+ minKey = minKeyCandidate;
+ }
+ }
+ }
+
+ // Sanity checks
+ if ( minKey == null ) {
+ // Every reader is empty and look-ahead buffer is empty (hasNext() should have said false)
+ throw new NoSuchElementException();
+ }
+
+ return minKey;
+ }
+
+ // Boring accessors
+
+ public String getLastKey() {
+ return lastKey;
+ }
+}
diff --git a/src/core/src/data/io/AbstractMVDataReader.java b/src/core/src/data/io/AbstractMVDataReader.java
new file mode 100644
index 0000000..3e63de1
--- /dev/null
+++ b/src/core/src/data/io/AbstractMVDataReader.java
@@ -0,0 +1,49 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io;
+
+/**
+ * Stream-oriented abstract reader from a particular data source.
+ * Memory footprint should not depends on readable line count nor next() call count.
+ *
+ * @author lpouzenc
+ */
+public abstract class AbstractMVDataReader implements MVDataReader {
+
+ protected String dataSourceName="(unknown source)";
+
+ /**
+ * Not supported (Readers are read-only).
+ */
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ /* (non-Javadoc)
+ * @see data.io.MVDataReader#getDataSourceName()
+ */
+ @Override
+ public String getDataSourceName() {
+ return dataSourceName;
+ }
+
+}
diff --git a/src/core/src/data/io/AbstractMVDataWriter.java b/src/core/src/data/io/AbstractMVDataWriter.java
new file mode 100644
index 0000000..454e8ce
--- /dev/null
+++ b/src/core/src/data/io/AbstractMVDataWriter.java
@@ -0,0 +1,70 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io;
+
+import java.util.Set;
+
+import data.MVDataEntry;
+
+/**
+ * Stream-oriented abstract writer from a particular data source.
+ * All derived writers should honor a dry-run mode.
+ *
+ * @author lpouzenc
+ */
+public abstract class AbstractMVDataWriter implements MVDataWriter {
+ //TODO : not so useful. Interface extraction was not a good idea ?
+
+ /**
+ * Dry-run mode flag (disabled by default)
+ */
+ protected boolean dryRun=false;
+
+ /* (non-Javadoc)
+ * @see data.io.MVDataWriter#isDryRun()
+ */
+ public boolean isDryRun() {
+ return dryRun;
+ }
+
+ /* (non-Javadoc)
+ * @see data.io.MVDataWriter#setDryRun(boolean)
+ */
+ public void setDryRun(boolean dryRun) {
+ this.dryRun = dryRun;
+ }
+
+ /* (non-Javadoc)
+ * @see data.io.MVDataWriter#insert(data.MVDataEntry)
+ */
+ @Override
+ public abstract void insert(MVDataEntry newEntry) throws Exception;
+ /* (non-Javadoc)
+ * @see data.io.MVDataWriter#update(data.MVDataEntry, data.MVDataEntry, java.util.Set)
+ */
+ @Override
+ public abstract void update(MVDataEntry updatedEntry, MVDataEntry originalEntry, Set<String> attrToUpdate) throws Exception;
+ /* (non-Javadoc)
+ * @see data.io.MVDataWriter#delete(data.MVDataEntry)
+ */
+ @Override
+ public abstract void delete(MVDataEntry existingEntry) throws Exception;
+}
diff --git a/src/core/src/data/io/MVDataReader.java b/src/core/src/data/io/MVDataReader.java
new file mode 100644
index 0000000..8a9871a
--- /dev/null
+++ b/src/core/src/data/io/MVDataReader.java
@@ -0,0 +1,39 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io;
+
+import java.util.Iterator;
+
+import data.MVDataEntry;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public interface MVDataReader extends Iterator<MVDataEntry>, Iterable<MVDataEntry>{
+
+ /**
+ * @return the dataSourceName
+ */
+ public String getDataSourceName();
+
+} \ No newline at end of file
diff --git a/src/core/src/data/io/MVDataWriter.java b/src/core/src/data/io/MVDataWriter.java
new file mode 100644
index 0000000..2f16fbc
--- /dev/null
+++ b/src/core/src/data/io/MVDataWriter.java
@@ -0,0 +1,45 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io;
+
+import java.util.Set;
+
+import data.MVDataEntry;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public interface MVDataWriter {
+
+ public boolean isDryRun();
+ public void setDryRun(boolean dryRun);
+
+ public void insert(MVDataEntry newEntry) throws Exception;
+
+ public void update(MVDataEntry updatedEntry,
+ MVDataEntry originalEntry, Set<String> attrToUpdate)
+ throws Exception;
+
+ public void delete(MVDataEntry existingEntry) throws Exception;
+
+} \ No newline at end of file
diff --git a/src/core/src/data/io/stub/StubDataReader.java b/src/core/src/data/io/stub/StubDataReader.java
new file mode 100644
index 0000000..ed91267
--- /dev/null
+++ b/src/core/src/data/io/stub/StubDataReader.java
@@ -0,0 +1,63 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io.stub;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import data.MVDataEntry;
+import data.io.AbstractMVDataReader;
+
+/**
+ * Stub reader implementation for automated tests.
+ *
+ * @author lpouzenc
+ */
+public class StubDataReader extends AbstractMVDataReader {
+
+ private final MVDataEntry fakeEntries[];
+ private int cursorRead;
+
+ public StubDataReader(String dataSourceName, MVDataEntry[] fakeEntries) {
+ this.dataSourceName = dataSourceName;
+ this.fakeEntries = fakeEntries.clone();
+ }
+
+ @Override
+ public Iterator<MVDataEntry> iterator() {
+ this.cursorRead = 0;
+ return this;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return cursorRead < fakeEntries.length;
+ }
+
+ @Override
+ public MVDataEntry next() {
+ if ( ! hasNext() ) {
+ throw new NoSuchElementException();
+ }
+ return fakeEntries[cursorRead++];
+ }
+
+}
diff --git a/src/core/src/data/io/stub/StubDataWriter.java b/src/core/src/data/io/stub/StubDataWriter.java
new file mode 100644
index 0000000..cd08e77
--- /dev/null
+++ b/src/core/src/data/io/stub/StubDataWriter.java
@@ -0,0 +1,104 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io.stub;
+
+import java.util.Set;
+
+import data.MVDataEntry;
+import data.io.AbstractMVDataWriter;
+
+/**
+ * Stub writer implementation for automated tests.
+ *
+ * @author lpouzenc
+ */
+public class StubDataWriter extends AbstractMVDataWriter {
+
+ enum OpKind { INSERT, UPDATE, DELETE };
+
+ private final int maxLogEntries;
+
+ private OpKind opLog[];
+ private MVDataEntry opData[];
+ private int cursorLog;
+
+ public StubDataWriter(int maxLogEntries) {
+ this.maxLogEntries = maxLogEntries;
+ this.opLog = new OpKind[maxLogEntries];
+ this.opData = new MVDataEntry[maxLogEntries];
+ }
+
+ @Override
+ public void insert(MVDataEntry newline) {
+ if ( cursorLog >= maxLogEntries) {
+ throw new IllegalStateException();
+ }
+ opLog[cursorLog]=OpKind.INSERT;
+ opData[cursorLog]=newline;
+ cursorLog++;
+ }
+
+ @Override
+ public void update(MVDataEntry updatedLine, MVDataEntry originalLine, Set<String> attrToUpdate) {
+ if ( cursorLog >= maxLogEntries) {
+ throw new IllegalStateException();
+ }
+ opLog[cursorLog]=OpKind.UPDATE;
+ opData[cursorLog]=updatedLine;
+ cursorLog++;
+ }
+
+ @Override
+ public void delete(MVDataEntry existingLine) {
+ if ( cursorLog >= maxLogEntries) {
+ throw new IllegalStateException();
+ }
+ opLog[cursorLog]=OpKind.DELETE;
+ opData[cursorLog]=existingLine;
+ cursorLog++;
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+
+ for (int i = 0; i < cursorLog; i++) {
+ buf.append(opLog[i] + ": " + opData[i] + "\n");
+ }
+
+ return buf.toString();
+ }
+
+ /**
+ * @return the opLog
+ */
+ public OpKind[] getOpLog() {
+ return opLog.clone();
+ }
+
+ /**
+ * @return the opData
+ */
+ public MVDataEntry[] getOpData() {
+ return opData.clone();
+ }
+
+}
diff --git a/src/core/src/sync/AbstractSyncTask.java b/src/core/src/sync/AbstractSyncTask.java
new file mode 100644
index 0000000..e2ae94d
--- /dev/null
+++ b/src/core/src/sync/AbstractSyncTask.java
@@ -0,0 +1,71 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package sync;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Abstract class to define a common base of all kind of synchronization algorithms.
+ *
+ * @author lpouzenc
+ */
+public abstract class AbstractSyncTask implements Callable<Boolean> {
+
+ /**
+ * Pretty task name to be inserted in log lines
+ */
+ protected String taskName="(unknown task)";
+
+ /**
+ * Dry-run mode flag (disabled by default)
+ */
+ protected boolean dryRun=false;
+
+ /**
+ * Main method that do the actual sync
+ */
+ public abstract Boolean call();
+
+
+ // Boring accessors
+
+ /**
+ * @return the dryRun mode status (enabled/disabled)
+ */
+ public boolean isDryRun() {
+ return dryRun;
+ }
+
+ /**
+ * @param dryRun the dryRun mode to set (enabled/disabled)
+ */
+ public void setDryRun(boolean dryRun) {
+ this.dryRun = dryRun;
+ }
+
+ /**
+ * @return the taskName
+ */
+ public String getTaskName() {
+ return taskName;
+ }
+
+}
diff --git a/src/main/.classpath b/src/main/.classpath
new file mode 100644
index 0000000..33bcbdb
--- /dev/null
+++ b/src/main/.classpath
@@ -0,0 +1,16 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="src" path="JUTests"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.6"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.junit.JUNIT_CONTAINER/4"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/SSSync_Core"/>
+ <classpathentry kind="lib" path="lib/log4j-1.2.17.jar"/>
+ <classpathentry kind="lib" path="lib/snakeyaml-1.11.jar">
+ <attributes>
+ <attribute name="javadoc_location" value="jar:platform:/resource/SSSync/lib/snakeyaml-1.11-javadoc.jar!/"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry combineaccessrules="false" kind="src" path="/SSSync_Connectors"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
diff --git a/src/main/.project b/src/main/.project
new file mode 100644
index 0000000..33a3a78
--- /dev/null
+++ b/src/main/.project
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+ <name>SSSync_Main</name>
+ <comment></comment>
+ <projects>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ <arguments>
+ </arguments>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
diff --git a/src/main/.settings/org.eclipse.jdt.core.prefs b/src/main/.settings/org.eclipse.jdt.core.prefs
new file mode 100644
index 0000000..8000cd6
--- /dev/null
+++ b/src/main/.settings/org.eclipse.jdt.core.prefs
@@ -0,0 +1,11 @@
+eclipse.preferences.version=1
+org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
+org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
+org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve
+org.eclipse.jdt.core.compiler.compliance=1.6
+org.eclipse.jdt.core.compiler.debug.lineNumber=generate
+org.eclipse.jdt.core.compiler.debug.localVariable=generate
+org.eclipse.jdt.core.compiler.debug.sourceFile=generate
+org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
+org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
+org.eclipse.jdt.core.compiler.source=1.6
diff --git a/src/main/JUTests/AllClientServerTests.java b/src/main/JUTests/AllClientServerTests.java
new file mode 100644
index 0000000..cef8ffd
--- /dev/null
+++ b/src/main/JUTests/AllClientServerTests.java
@@ -0,0 +1,18 @@
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+import data.io.ldap.LDAPDataReaderTest;
+import data.io.ldap.LDAPDataWriterTest;
+import data.io.sql.SQLRelDataReaderTest;
+
+
+@RunWith(Suite.class)
+@SuiteClasses({
+ // SSSync_Connectors
+ LDAPDataReaderTest.class, LDAPDataWriterTest.class,
+ SQLRelDataReaderTest.class,
+})
+public class AllClientServerTests {
+
+}
diff --git a/src/main/JUTests/AllLocalTests.java b/src/main/JUTests/AllLocalTests.java
new file mode 100644
index 0000000..bc9019d
--- /dev/null
+++ b/src/main/JUTests/AllLocalTests.java
@@ -0,0 +1,29 @@
+
+
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.junit.runners.Suite.SuiteClasses;
+
+import sync.BasicSyncTaskTest;
+
+import conf.SSSyncConfParserTest;
+
+import data.MVDataEntryTest;
+import data.io.SafeDataReaderTest;
+import data.io.csv.CSVDataReaderTest;
+import data.io.filters.MVDataCombinerTest;
+
+@RunWith(Suite.class)
+@SuiteClasses( {
+ // SSSync
+ SSSyncConfParserTest.class,
+ SafeDataReaderTest.class,
+ BasicSyncTaskTest.class,
+ // SSSync_Connectors (only local)
+ CSVDataReaderTest.class,
+ // SSSync_Core
+ MVDataEntryTest.class, MVDataCombinerTest.class,
+ } )
+public class AllLocalTests {
+
+}
diff --git a/src/main/JUTests/conf/SSSyncConfParserTest.java b/src/main/JUTests/conf/SSSyncConfParserTest.java
new file mode 100644
index 0000000..100df16
--- /dev/null
+++ b/src/main/JUTests/conf/SSSyncConfParserTest.java
@@ -0,0 +1,69 @@
+package conf;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URL;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.Yaml;
+
+public class SSSyncConfParserTest {
+
+ private File currentFolder;
+
+ @Before
+ public void setup() {
+ URL main = SSSyncConfParserTest.class.getResource("SSSyncConfParserTest.class");
+ if (!"file".equalsIgnoreCase(main.getProtocol()))
+ throw new IllegalStateException("This class is not stored in a file");
+ currentFolder = new File(main.getPath()).getParentFile();
+ }
+
+ @Test
+ public void loadConfigTest() throws Exception {
+
+ String expectedMain = readEntireFile(new File(currentFolder, "testExpectedMain.yaml"));
+ String expectedConn = readEntireFile(new File(currentFolder, "testExpectedConn.yaml"));
+ String mainConfigFile = new File(currentFolder, "testMain.yaml").getAbsolutePath();
+ String connConfigFile = new File(currentFolder, "testConn.yaml").getAbsolutePath();
+
+ // Loading (config => beans)
+ ConfigRootBean confMain = SSSyncConfParser.loadMainConfig(mainConfigFile);
+ ConfigConnectionsBean confConn = SSSyncConfParser.loadConnConfig(connConfigFile);
+
+
+ System.out.println(confMain);
+ System.out.println(confConn);
+
+ // Dumping (beans => config)
+ DumperOptions options = new DumperOptions();
+ options.setDefaultFlowStyle(DumperOptions.FlowStyle.BLOCK);
+ Yaml yamlDump = new Yaml(options);
+ String dumpMain = yamlDump.dump(confMain);
+ String dumpConn = yamlDump.dump(confConn);
+
+ // Checking that everything is kept
+ assertEquals(expectedMain, dumpMain);
+ assertEquals(expectedConn, dumpConn);
+ }
+
+ private static String readEntireFile(File file) throws IOException {
+ FileReader in = new FileReader(file);
+ StringBuilder contents = new StringBuilder((int) file.length());
+ char[] buffer = new char[4096];
+ int read = 0;
+ do {
+ contents.append(buffer, 0, read);
+ read = in.read(buffer);
+ } while (read >= 0);
+ in.close();
+
+ return contents.toString();
+ }
+
+}
diff --git a/src/main/JUTests/conf/testConn.yaml b/src/main/JUTests/conf/testConn.yaml
new file mode 100644
index 0000000..c41063c
--- /dev/null
+++ b/src/main/JUTests/conf/testConn.yaml
@@ -0,0 +1,19 @@
+# This file contains credentials (should be readable only by SSSync)
+connections:
+ - id : ora_1
+ type: jdbc
+ dbms: oracle
+ ress: gest
+ host: ora.univ-jfc.fr
+ port: 1521
+ user: GRHUM
+ pass: secret
+ db : GHRUM
+
+ - id : ldap_1
+ type: ldap
+ host: localhost
+ port: 389
+ bind: uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr
+ pass: secret
+
diff --git a/src/main/JUTests/conf/testExpectedConn.yaml b/src/main/JUTests/conf/testExpectedConn.yaml
new file mode 100644
index 0000000..4cb3421
--- /dev/null
+++ b/src/main/JUTests/conf/testExpectedConn.yaml
@@ -0,0 +1,22 @@
+!!conf.ConfigConnectionsBean
+connections:
+- bind: null
+ db: GHRUM
+ dbms: oracle
+ host: ora.univ-jfc.fr
+ id: ora_1
+ pass: secret
+ port: 1521
+ ress: gest
+ type: jdbc
+ user: GRHUM
+- bind: uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr
+ db: null
+ dbms: null
+ host: localhost
+ id: ldap_1
+ pass: secret
+ port: 389
+ ress: null
+ type: ldap
+ user: null
diff --git a/src/main/JUTests/conf/testExpectedMain.yaml b/src/main/JUTests/conf/testExpectedMain.yaml
new file mode 100644
index 0000000..dd00aef
--- /dev/null
+++ b/src/main/JUTests/conf/testExpectedMain.yaml
@@ -0,0 +1,70 @@
+!!conf.ConfigRootBean
+globals:
+ maxExecTime: 3
+tasks:
+- destination:
+ attr: uid
+ base: ou=people,dc=univ-jfc,dc=fr
+ conn: ldap_1
+ kind: ldap
+ mode: null
+ name: LDAP de test, ou=people
+ path: null
+ query: null
+ name: People sync
+ opLimits:
+ delete: 10
+ insert: 100
+ update: 10
+ skipEntryDelete: false
+ skipReadErrors: false
+ sources:
+ - attr: null
+ base: null
+ conn: ora_1
+ kind: sql
+ mode: PRIMARY_SOURCE
+ name: GHRUM, comptes et personnes
+ path: null
+ query: people.sql
+ - attr: null
+ base: null
+ conn: null
+ kind: csv
+ mode: MERGE_APPEND
+ name: CSV personnes additionnelles
+ path: people_append.csv
+ query: null
+ - attr: null
+ base: null
+ conn: null
+ kind: sorted_csv
+ mode: MERGE_REPLACE
+ name: CSV correctifs personnes
+ path: people_replace.csv
+ query: null
+- destination:
+ attr: supannEntiteAffectation
+ base: ou=structures,dc=univ-jfc,dc=fr
+ conn: ldap_1
+ kind: ldap
+ mode: null
+ name: LDAP de test, ou=structures
+ path: null
+ query: null
+ name: Structure sync
+ opLimits:
+ delete: 10
+ insert: 10
+ update: 10
+ skipEntryDelete: true
+ skipReadErrors: true
+ sources:
+ - attr: null
+ base: null
+ conn: ora_1
+ kind: sql
+ mode: PRIMARY_SOURCE
+ name: GHRUM, structures
+ path: null
+ query: structures.sql
diff --git a/src/main/JUTests/conf/testMain.yaml b/src/main/JUTests/conf/testMain.yaml
new file mode 100644
index 0000000..39350b2
--- /dev/null
+++ b/src/main/JUTests/conf/testMain.yaml
@@ -0,0 +1,54 @@
+# This YAML file describe all synchronization tasks, with their readers and writers
+
+globals:
+ maxExecTime: 3
+
+tasks:
+ - name: People sync
+ opLimits:
+ insert: 100
+ update: 10
+ delete: 10
+ sources:
+ - name: GHRUM, comptes et personnes
+ kind: sql
+ conn: ora_1
+ mode: PRIMARY_SOURCE
+ query: people.sql
+
+ - name: CSV personnes additionnelles
+ kind: csv
+ mode: MERGE_APPEND
+ path: people_append.csv
+
+ - name: CSV correctifs personnes
+ kind: sorted_csv
+ mode: MERGE_REPLACE
+ path: people_replace.csv
+
+ destination:
+ name: LDAP de test, ou=people
+ kind: ldap
+ conn: ldap_1
+ attr: uid
+ base: ou=people,dc=univ-jfc,dc=fr
+
+ - name: Structure sync
+ sources:
+ - name: GHRUM, structures
+ kind: sql
+ conn: ora_1
+ mode: PRIMARY_SOURCE
+ query: structures.sql
+ destination:
+ name: LDAP de test, ou=structures
+ kind: ldap
+ conn: ldap_1
+ attr: supannEntiteAffectation
+ base: ou=structures,dc=univ-jfc,dc=fr
+ skipEntryDelete: true
+ skipReadErrors: true
+ opLimits:
+ insert: 10
+ update: 10
+ delete: 10 \ No newline at end of file
diff --git a/src/main/JUTests/data/io/SafeDataReaderTest.java b/src/main/JUTests/data/io/SafeDataReaderTest.java
new file mode 100644
index 0000000..427004b
--- /dev/null
+++ b/src/main/JUTests/data/io/SafeDataReaderTest.java
@@ -0,0 +1,51 @@
+package data.io;
+
+import static org.junit.Assert.*;
+
+import java.util.Iterator;
+
+import org.apache.log4j.PropertyConfigurator;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import data.MVDataEntry;
+import data.io.stub.StubDataReader;
+
+public class SafeDataReaderTest {
+
+ private static final String LOG_PROPERTIES_FILE = "conf/log4j.properties";
+
+ @BeforeClass
+ public static void setup() {
+ PropertyConfigurator.configure(LOG_PROPERTIES_FILE);
+ }
+
+ @Test
+ public void testNoErrors() {
+ MVDataEntry testEntries[] = new MVDataEntry[5];
+ for (int i=0;i<5;i++) {
+ testEntries[i] = new MVDataEntry("line"+(i+1));
+ testEntries[i].put("attr1", "value"+(i+1));
+ }
+
+ StubDataReader src = new StubDataReader("testNoSkipErrors_src", testEntries);
+ StubDataReader expected = new StubDataReader("testNoSkipErrors_expected", testEntries);
+
+ SafeDataReader reader = new SafeDataReader(src, false);
+
+ // Test twice to check if asking a new iterator "rewinds" correctly
+ for (int i=0;i<2;i++) {
+ //System.out.println("Loop " + (i+1));
+ Iterator<MVDataEntry> readerIt = reader.iterator();
+ for ( MVDataEntry e: expected) {
+ assertTrue(readerIt.hasNext());
+ MVDataEntry r = readerIt.next();
+ //System.out.println(e + " / " + r);
+ assertEquals(e, r);
+ }
+ assertFalse(readerIt.hasNext());
+ }
+ }
+
+ //TODO Real tests with messy input readers (null values, exception, hasNext/next() incoherence)
+}
diff --git a/src/main/JUTests/sync/BasicSyncTaskTest.java b/src/main/JUTests/sync/BasicSyncTaskTest.java
new file mode 100644
index 0000000..88d9c98
--- /dev/null
+++ b/src/main/JUTests/sync/BasicSyncTaskTest.java
@@ -0,0 +1,129 @@
+package sync;
+
+import static org.junit.Assert.*;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.log4j.PropertyConfigurator;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import data.MVDataEntry;
+import data.filters.MVDataCombiner;
+import data.filters.MVDataCombiner.MVDataCombineMode;
+import data.io.MVDataReader;
+import data.io.SafeDataReader;
+import data.io.csv.CSVDataReader;
+import data.io.stub.StubDataReader;
+import data.io.stub.StubDataWriter;
+
+public class BasicSyncTaskTest {
+
+ private static final String LOG_PROPERTIES_FILE = "conf/log4j.properties";
+
+ @BeforeClass
+ public static void setup() {
+ PropertyConfigurator.configure(LOG_PROPERTIES_FILE);
+ }
+
+ @Test
+ public void test() throws IOException {
+
+ // Input flows setup
+ MVDataEntry[] fakeEntries1 = new MVDataEntry[5];
+ fakeEntries1[0] = new MVDataEntry("line1");
+ fakeEntries1[0].put("hello", "world");
+
+ fakeEntries1[1] = new MVDataEntry("line2");
+ fakeEntries1[1].put("bla", "hidden");
+ fakeEntries1[1].put("hello", "merged");
+
+ fakeEntries1[2] = new MVDataEntry("line3");
+ fakeEntries1[2].put("hello", "world");
+
+ fakeEntries1[3] = new MVDataEntry("line4");
+ fakeEntries1[3].put("hello", "world");
+
+ fakeEntries1[4] = new MVDataEntry("line5");
+ fakeEntries1[4].put("hello", "world");
+
+
+ MVDataEntry[] fakeEntries2 = new MVDataEntry[3];
+ fakeEntries2[0] = new MVDataEntry("line1");
+ fakeEntries2[0].put("hello", "world");
+
+ fakeEntries2[1] = new MVDataEntry("line2");
+ fakeEntries2[1].put("bla", "replaced");
+
+ fakeEntries2[2] = new MVDataEntry("line3");
+ fakeEntries2[2].put("hello", "world");
+
+
+ MVDataEntry[] fakeEntries3 = new MVDataEntry[5];
+ fakeEntries3[0] = new MVDataEntry("line2");
+ fakeEntries3[0].put("hello", "world");
+ fakeEntries3[0].put("extra", "to be preserved");
+
+ fakeEntries3[1] = new MVDataEntry("line2b");
+ fakeEntries3[1].put("to be", "removed", null);
+
+ fakeEntries3[2] = new MVDataEntry("line4");
+ fakeEntries3[2].put("hello", "world");
+ fakeEntries3[2].put("extra", "to be preserved");
+
+ fakeEntries3[3] = new MVDataEntry("line5");
+ fakeEntries3[3].splitAndPut("hello", "too;much;world", ";");
+
+ fakeEntries3[4] = new MVDataEntry("line6");
+ fakeEntries3[4].put("to be", "removed");
+
+ StubDataReader fakeReader1 = new StubDataReader("testSrc1", fakeEntries1);
+ StubDataReader fakeReader2 = new StubDataReader("testSrc3", fakeEntries2);
+ StubDataReader fakeReader3 = new StubDataReader("testDst", fakeEntries3);
+
+ MVDataReader readers[] = new MVDataReader[]{
+ new SafeDataReader(fakeReader1,false),
+ new SafeDataReader(
+ new CSVDataReader("testSrc2",
+ new StringReader(CSVDataReader.CSV_DEMO),
+ false
+ ), false
+ ),
+ new SafeDataReader(fakeReader2,false),
+ };
+
+ MVDataCombineMode mergeModes[] = new MVDataCombineMode[]{
+ MVDataCombineMode.PRIMARY_SOURCE,
+ MVDataCombineMode.MERGE_APPEND,
+ MVDataCombineMode.MERGE_REPLACE,
+ };
+
+ MVDataReader srcReader = new MVDataCombiner("testSrcComb", readers, mergeModes);
+ MVDataReader dstReader = fakeReader3;
+
+ // Output flow setup
+ StubDataWriter dstWriter = new StubDataWriter(10);
+
+ // Data sync'er initialization
+ BasicSyncTask task = new BasicSyncTask("task1", false, srcReader, dstReader, dstWriter);
+ task.setOperationLimits(10,10,10);
+
+ // Data sync'er run
+ assertTrue(task.call());
+
+ // Expected outputs
+ String expectedDstOps =
+ "INSERT: {key=line1, attrValPairs={hello=[world], attr2=[csv1], from=[csv1, csv1bis]}}\n" +
+ "UPDATE: {key=line2, attrValPairs={hello=[the, merged, world, all], bla=[replaced]}}\n" +
+ "DELETE: {key=line2b, attrValPairs={to be=[removed]}}\n" +
+ "INSERT: {key=line3, attrValPairs={hello=[world]}}\n" +
+ // Line 4 must not be updated !
+ "UPDATE: {key=line5, attrValPairs={hello=[world]}}\n" +
+ "DELETE: {key=line6, attrValPairs={to be=[removed]}}\n";
+
+ // Check results
+ assertEquals(expectedDstOps, dstWriter.toString());
+ }
+
+}
diff --git a/src/main/build.xml b/src/main/build.xml
new file mode 100644
index 0000000..8847365
--- /dev/null
+++ b/src/main/build.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- WARNING: Eclipse auto-generated file.
+ Any modifications will be overwritten.
+ To include a user specific buildfile here, simply create one in the same
+ directory with the processing instruction <?eclipse.ant.import?>
+ as the first entry and export the buildfile again. -->
+<project basedir="." default="build" name="SSSync_Main">
+ <property environment="env"/>
+ <property name="ECLIPSE_HOME" value="../../../../../../usr/lib/eclipse"/>
+ <property name="SSSync_Core.location" value="../core"/>
+ <property name="SSSync_Connectors.location" value="../connectors"/>
+ <property name="debuglevel" value="source,lines,vars"/>
+ <property name="target" value="1.6"/>
+ <property name="source" value="1.6"/>
+ <path id="JUnit 4.libraryclasspath">
+ <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.junit_4.8.2.dist/junit.jar"/>
+ <pathelement location="../../../../../../usr/share/eclipse/dropins/jdt/plugins/org.hamcrest.core_1.1.0.jar"/>
+ </path>
+ <path id="SSSync_Core.classpath">
+ <pathelement location="${SSSync_Core.location}/bin"/>
+ <pathelement location="${SSSync_Core.location}/lib/guava-16.0.1.jar"/>
+ <path refid="JUnit 4.libraryclasspath"/>
+ </path>
+ <path id="SSSync_Connectors.classpath">
+ <pathelement location="${SSSync_Connectors.location}/bin"/>
+ <path refid="SSSync_Core.classpath"/>
+ <path refid="JUnit 4.libraryclasspath"/>
+ <pathelement location="${SSSync_Connectors.location}/lib/commons-csv-1.0-SNAPSHOT.jar"/>
+ <pathelement location="${SSSync_Connectors.location}/lib/ojdbc6.jar"/>
+ <pathelement location="${SSSync_Connectors.location}/lib/mysql-connector-java-5.1.31-bin.jar"/>
+ <pathelement location="${SSSync_Connectors.location}/lib/unboundid-ldapsdk-se.jar"/>
+ </path>
+ <path id="SSSync_Main.classpath">
+ <pathelement location="bin"/>
+ <path refid="JUnit 4.libraryclasspath"/>
+ <path refid="SSSync_Core.classpath"/>
+ <pathelement location="lib/log4j-1.2.17.jar"/>
+ <pathelement location="lib/snakeyaml-1.11.jar"/>
+ <path refid="SSSync_Connectors.classpath"/>
+ </path>
+ <target name="init">
+ <mkdir dir="bin"/>
+ <copy includeemptydirs="false" todir="bin">
+ <fileset dir="src">
+ <exclude name="**/*.java"/>
+ </fileset>
+ </copy>
+ <copy includeemptydirs="false" todir="bin">
+ <fileset dir="JUTests">
+ <exclude name="**/*.java"/>
+ </fileset>
+ </copy>
+ </target>
+ <target name="clean">
+ <delete dir="bin"/>
+ </target>
+ <target depends="clean" name="cleanall">
+ <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="clean"/>
+ <ant antfile="build.xml" dir="${SSSync_Connectors.location}" inheritAll="false" target="clean"/>
+ </target>
+ <target depends="build-subprojects,build-project" name="build"/>
+ <target name="build-subprojects">
+ <ant antfile="build.xml" dir="${SSSync_Core.location}" inheritAll="false" target="build-project">
+ <propertyset>
+ <propertyref name="build.compiler"/>
+ </propertyset>
+ </ant>
+ <ant antfile="build.xml" dir="${SSSync_Connectors.location}" inheritAll="false" target="build-project">
+ <propertyset>
+ <propertyref name="build.compiler"/>
+ </propertyset>
+ </ant>
+ </target>
+ <target depends="init" name="build-project">
+ <echo message="${ant.project.name}: ${ant.file}"/>
+ <javac debug="true" debuglevel="${debuglevel}" destdir="bin" includeantruntime="false" source="${source}" target="${target}">
+ <src path="src"/>
+ <src path="JUTests"/>
+ <classpath refid="SSSync_Main.classpath"/>
+ </javac>
+ </target>
+ <target description="Build all projects which reference this project. Useful to propagate changes." name="build-refprojects"/>
+ <target description="copy Eclipse compiler jars to ant lib directory" name="init-eclipse-compiler">
+ <copy todir="${ant.library.dir}">
+ <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/>
+ </copy>
+ <unzip dest="${ant.library.dir}">
+ <patternset includes="jdtCompilerAdapter.jar"/>
+ <fileset dir="${ECLIPSE_HOME}/plugins" includes="org.eclipse.jdt.core_*.jar"/>
+ </unzip>
+ </target>
+ <target description="compile project with Eclipse compiler" name="build-eclipse-compiler">
+ <property name="build.compiler" value="org.eclipse.jdt.core.JDTCompilerAdapter"/>
+ <antcall target="build"/>
+ </target>
+ <target name="SSSync">
+ <java classname="SSSync" failonerror="true" fork="yes">
+ <classpath refid="SSSync_Main.classpath"/>
+ </java>
+ </target>
+</project>
diff --git a/src/main/conf/connections.yaml b/src/main/conf/connections.yaml
new file mode 100644
index 0000000..1918d02
--- /dev/null
+++ b/src/main/conf/connections.yaml
@@ -0,0 +1,18 @@
+# This file contains credentials (should be readable only by SSSync)
+connections:
+ - id : mysql_1
+ type: jdbc
+ dbms: mysql
+ host: localhost
+ port: 3306
+ user: root
+ pass: secret
+ db : sssync
+
+ - id : ldap_1
+ type: ldap
+ host: localhost
+ port: 389
+ bind: uid=ldapadmin,ou=specialUsers,dc=univ-jfc,dc=fr
+ pass: secret
+
diff --git a/src/main/conf/log4j.properties b/src/main/conf/log4j.properties
new file mode 100644
index 0000000..6dccde8
--- /dev/null
+++ b/src/main/conf/log4j.properties
@@ -0,0 +1,29 @@
+#
+# our log4j properties / configuration file
+#
+# STDOUT appender
+log4j.appender.STDOUT=org.apache.log4j.ConsoleAppender
+log4j.appender.STDOUT.layout=org.apache.log4j.PatternLayout
+log4j.appender.STDOUT.layout.ConversionPattern=%d %p [%t] %C{1} - %m\n
+
+
+# Normal operation mode
+log4j.category.SSSync=INFO, STDOUT
+# Configuration dump
+#log4j.category.SSSync=DEBUG, STDOUT
+
+# Normal operation mode (currently nothing more in DEBUG or TRACE modes)
+log4j.category.data.io.SafeDataReader=INFO, STDOUT
+
+# Normal operation mode
+#log4j.category.sync.BasicSyncTask=INFO, STDOUT
+# Trace insert/update/delete operation
+log4j.category.sync.BasicSyncTask=DEBUG, STDOUT
+# Trace every key comparison
+#log4j.category.sync.BasicSyncTask=TRACE, STDOUT
+
+# Keep silent about memory and GC
+log4j.category.utils.JVMStatsDumper=INFO, STDOUT
+# Trace memory usage/GC + dump configuration
+#log4j.category.utils.JVMStatsDumper=DEBUG, STDOUT
+
diff --git a/src/main/conf/queries/people.sql b/src/main/conf/queries/people.sql
new file mode 100644
index 0000000..ab66d5f
--- /dev/null
+++ b/src/main/conf/queries/people.sql
@@ -0,0 +1,5 @@
+SELECT
+ p.*,
+ "person;posixAccount;top" as objectClass
+FROM sssync.people p
+ORDER BY 1 ASC;
diff --git a/src/main/conf/queries/structures.sql b/src/main/conf/queries/structures.sql
new file mode 100644
index 0000000..626273c
--- /dev/null
+++ b/src/main/conf/queries/structures.sql
@@ -0,0 +1,5 @@
+SELECT
+ s.*,
+ "supannEntite;organizationalUnit;top" as objectClass
+FROM sssync.structures s
+ORDER BY 1 ASC;
diff --git a/src/main/conf/sssync.yaml b/src/main/conf/sssync.yaml
new file mode 100644
index 0000000..b285a37
--- /dev/null
+++ b/src/main/conf/sssync.yaml
@@ -0,0 +1,56 @@
+# This YAML file describe all synchronization tasks, with their readers and writers
+
+globals:
+ maxExecTime: 3 # minutes
+
+tasks:
+ - name: People sync
+ opLimits:
+ insert: 300
+ update: 300
+ delete: 300
+ sources:
+ - name: GHRUM, comptes et personnes
+ kind: sql
+ conn: mysql_1
+ mode: PRIMARY_SOURCE
+ query: conf/queries/people.sql
+
+ - name: CSV personnes additionnelles
+ kind: csv
+ mode: MERGE_APPEND
+ path: data/people_append.csv
+
+ - name: CSV correctifs personnes
+ kind: csv
+ mode: MERGE_REPLACE
+ path: data/people_replace.csv
+
+ destination:
+ name: LDAP de test, ou=people
+ kind: ldap
+ conn: ldap_1
+ attr: uid
+ base: ou=people,dc=univ-jfc,dc=fr
+
+ - name: Structure sync
+ opLimits:
+ insert: 10
+ update: 10
+ delete: 10
+ sources:
+ - name: GHRUM, structures
+ kind: sql
+ conn: mysql_1
+ mode: PRIMARY_SOURCE
+ query: conf/queries/structures.sql
+
+ destination:
+ name: LDAP de test, ou=structures
+ kind: ldap
+ conn: ldap_1
+ attr: supannCodeEntite
+ base: ou=structures,dc=univ-jfc,dc=fr
+
+ skipEntryDelete: true
+ skipReadErrors: true \ No newline at end of file
diff --git a/src/main/data/people_append.csv b/src/main/data/people_append.csv
new file mode 100644
index 0000000..dc526ff
--- /dev/null
+++ b/src/main/data/people_append.csv
@@ -0,0 +1 @@
+lpouzenc,cn,Second-prénom \ No newline at end of file
diff --git a/src/main/data/people_replace.csv b/src/main/data/people_replace.csv
new file mode 100644
index 0000000..372ed67
--- /dev/null
+++ b/src/main/data/people_replace.csv
@@ -0,0 +1,3 @@
+lpouzenc,loginShell,/bin/ksh
+,,
+
diff --git a/src/main/lib/log4j-1.2.17.jar b/src/main/lib/log4j-1.2.17.jar
new file mode 100644
index 0000000..068867e
--- /dev/null
+++ b/src/main/lib/log4j-1.2.17.jar
Binary files differ
diff --git a/src/main/lib/snakeyaml-1.11-javadoc.jar b/src/main/lib/snakeyaml-1.11-javadoc.jar
new file mode 100644
index 0000000..bac2a05
--- /dev/null
+++ b/src/main/lib/snakeyaml-1.11-javadoc.jar
Binary files differ
diff --git a/src/main/lib/snakeyaml-1.11.jar b/src/main/lib/snakeyaml-1.11.jar
new file mode 100644
index 0000000..3e237cd
--- /dev/null
+++ b/src/main/lib/snakeyaml-1.11.jar
Binary files differ
diff --git a/src/main/src/SSSync.java b/src/main/src/SSSync.java
new file mode 100644
index 0000000..422c31e
--- /dev/null
+++ b/src/main/src/SSSync.java
@@ -0,0 +1,208 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.log4j.PropertyConfigurator;
+
+import conf.ConfigConnectionsBean;
+import conf.ConfigGlobalsBean;
+import conf.ConfigRootBean;
+import conf.SSSyncConfParser;
+import conf.SSSyncConnectionsFactory;
+import conf.SSSyncTasksFactory;
+import data.io.ConnectionsHolder;
+
+import sync.BasicSyncTask;
+import utils.JVMStatsDumper;
+
+/**
+ * Main class for Simple and Stupid Sync'er
+ *
+ * @author lpouzenc
+ */
+public class SSSync {
+ private static final Logger logger = Logger.getLogger(SSSync.class.getName());
+
+ private static final String LOG_PROPERTIES_FILE = "conf/log4j.properties";
+ private static final String CONFIG_MAIN_FILE = "conf/sssync.yaml";
+ private static final String CONFIG_CONN_FILE = "conf/connections.yaml";
+
+ private static final int ERR_SUCCESS = 0;
+ private static final int ERR_CONFIG_PARSE_ERROR = 1;
+ private static final int ERR_CONN_INIT_ERROR = 2;
+ private static final int ERR_TASK_INIT_ERROR = 3;
+ private static final int ERR_DRYRUN_FAILURE = 4;
+ private static final int ERR_REALRUN_FAILURE = 5;
+ //TODO private static final int ERR_MAXTIME_REACHED = 6;
+
+ /**
+ * Main entry point. Takes care of cmdline parsing, config files interpretation,
+ * tasks setup and start.
+ *
+ * @param args
+ */
+ public static void main(String[] args) {
+ // log4j setup (first thing to do)
+ PropertyConfigurator.configure(LOG_PROPERTIES_FILE);
+ logger.info("Program start (user: '" + System.getProperty("user.name") +
+ "', cwd: '" + System.getProperty("user.dir") + "')");
+
+ //TODO use cmdline args for config file path
+ String mainConfigFile = CONFIG_MAIN_FILE;
+ String connConfigFile = CONFIG_CONN_FILE;
+
+ // Config parsing
+ ConfigRootBean confMain = null;
+ ConfigConnectionsBean confConn = null;
+ try {
+ confMain = SSSyncConfParser.loadMainConfig(mainConfigFile);
+ confConn = SSSyncConfParser.loadConnConfig(connConfigFile);
+ } catch (Exception e) {
+ logger.fatal("Exception while loading configuration", e);
+ end(ERR_CONFIG_PARSE_ERROR);
+ }
+ ConfigGlobalsBean confGlobals = confMain.getGlobals();
+
+ // Config dump if DEBUG level (or finer)
+ if ( !logger.getLevel().isGreaterOrEqual(Level.INFO) ) {
+ logger.debug("Current connection configuration :\n" + confConn);
+ logger.debug("Current main configuration :\n" + confMain);
+ }
+
+ // Connections init
+ logger.info("Connections initialization");
+ ConnectionsHolder connections = null;
+ try {
+ connections = SSSyncConnectionsFactory.setupConnections(confConn);
+ } catch (Exception e) {
+ logger.fatal("Exception while establishing connections", e);
+ end(ERR_CONN_INIT_ERROR);
+ }
+
+ // Suggest garbage collector to forget our passwords since we are connected
+ confConn=null;
+ System.gc();
+ JVMStatsDumper.logMemoryUsage();
+
+
+ // Tasks init
+ logger.info("Tasks initialization");
+ List<BasicSyncTask> tasks = null;
+ try {
+ tasks = SSSyncTasksFactory.setupTasks(connections, confMain);
+ } catch (Exception e) {
+ logger.fatal("Exception during tasks initialization", e);
+ end(ERR_TASK_INIT_ERROR);
+ }
+
+ logger.info("Tasks are ready to start");
+ JVMStatsDumper.logMemoryUsage();
+
+
+ // Tasks first (dry) run
+ if ( ! SSSync.safeTaskRun(tasks, confGlobals.getMaxExecTime(), true) ) {
+ logger.error("Dry-run pass has shown problems, skipping real synchronization");
+ end(ERR_DRYRUN_FAILURE);
+ }
+
+ // Tasks second (real) run
+ if ( SSSync.safeTaskRun(tasks, confGlobals.getMaxExecTime(), false) ) {
+ logger.error("Real-run pass has shown problems, data could be messed up !");
+ end(ERR_REALRUN_FAILURE);
+ }
+
+ // Clean-up
+ try {
+ connections.close();
+ } catch (IOException e) {
+ logger.info("Problem during connections closing");
+ }
+
+ // Normal exit
+ end(ERR_SUCCESS);
+ }
+
+ /**
+ * Method to run safely a sequence of tasks within a given time period.
+ * In a separate thread, it runs all the tasks sequentially.
+ *
+ * @param list
+ * @param timeOutInMinute
+ * @return
+ * @throws ExecutionException
+ * @throws InterruptedException
+ */
+ private static boolean safeTaskRun(List<BasicSyncTask> list, long timeOutInMinute, boolean dryRun) {
+ ExecutorService executor = Executors.newSingleThreadExecutor();
+ List<Future<Boolean>> results;
+ boolean aborted = false;
+
+ logger.info("Starting " + (dryRun?"dry-run":"real-run") + " synchronization pass");
+
+ for ( BasicSyncTask t : list ) {
+ t.setDryRun(dryRun);
+ }
+
+ try {
+ results = executor.invokeAll(list, timeOutInMinute, TimeUnit.MINUTES);
+ // Join all tasks, seeking for an unsuccessful execution
+ for (Future<Boolean> r: results) {
+ if ( ! r.get() ) {
+ aborted = true;
+ }
+ }
+ } catch (CancellationException e) {
+ logger.fatal("Global maximum execution time exhausted, aborting tasks !");
+ aborted = true;
+ } catch (InterruptedException e) {
+ logger.fatal("Worker thread for task execution was interrupted", e);
+ aborted = true;
+ } catch (ExecutionException e) {
+ logger.error("Exception during tasks execution", e.getCause());
+ aborted = true;
+ }
+
+ JVMStatsDumper.logMemoryUsage();
+ executor.shutdown();
+
+ return !aborted;
+ }
+
+ /**
+ * Helper function to always log the end of program
+ * @param result
+ */
+ private static void end(int result) {
+ JVMStatsDumper.logGCStats();
+ logger.info("Program end (result code: " + result + ")");
+ System.exit(result);
+ }
+
+}
diff --git a/src/main/src/conf/ConfigConnectionBean.java b/src/main/src/conf/ConfigConnectionBean.java
new file mode 100644
index 0000000..b43b56f
--- /dev/null
+++ b/src/main/src/conf/ConfigConnectionBean.java
@@ -0,0 +1,111 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+import data.io.sql.SQLConnectionWrapper.DBMSType;
+
+/**
+ * Generated Configuration Bean
+ */
+public class ConfigConnectionBean {
+
+ public enum ConnectionType { jdbc, ldap }
+
+ private String id;
+ private ConnectionType type;
+ private DBMSType dbms;
+ private String ress;
+ private String host;
+ private int port;
+ private String user;
+ private String bind;
+ private String pass;
+ private String db;
+
+ public String getId() {
+ return id;
+ }
+ public void setId(String id) {
+ this.id = id;
+ }
+ public ConnectionType getType() {
+ return type;
+ }
+ public void setType(ConnectionType type) {
+ this.type = type;
+ }
+ public DBMSType getDbms() {
+ return dbms;
+ }
+ public void setDbms(DBMSType dbms) {
+ this.dbms = dbms;
+ }
+ public String getRess() {
+ return ress;
+ }
+ public void setRess(String ress) {
+ this.ress = ress;
+ }
+ public String getHost() {
+ return host;
+ }
+ public void setHost(String host) {
+ this.host = host;
+ }
+ public int getPort() {
+ return port;
+ }
+ public void setPort(int port) {
+ this.port = port;
+ }
+ public String getUser() {
+ return user;
+ }
+ public void setUser(String user) {
+ this.user = user;
+ }
+ public String getBind() {
+ return bind;
+ }
+ public void setBind(String bind) {
+ this.bind = bind;
+ }
+ public String getPass() {
+ return pass;
+ }
+ public void setPass(String pass) {
+ this.pass = pass;
+ }
+ public String getDb() {
+ return db;
+ }
+ public void setDb(String db) {
+ this.db = db;
+ }
+
+ @Override
+ public String toString() {
+ return "ConfigConnectionBean [id=" + id + ", type=" + type + ", dbms=" + dbms
+ + ", ress=" + ress + ", host=" + host + ", port=" + port
+ + ", user=" + user + ", bind=" + bind + ", pass=(obfuscated)]";
+ }
+
+}
diff --git a/src/main/src/conf/ConfigConnectionsBean.java b/src/main/src/conf/ConfigConnectionsBean.java
new file mode 100644
index 0000000..9fb034b
--- /dev/null
+++ b/src/main/src/conf/ConfigConnectionsBean.java
@@ -0,0 +1,45 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+import java.util.List;
+
+/**
+ * Generated Configuration Bean
+ */
+public class ConfigConnectionsBean {
+
+ private List<ConfigConnectionBean> connections;
+
+ public List<ConfigConnectionBean> getConnections() {
+ return connections;
+ }
+
+ public void setConnections(List<ConfigConnectionBean> connections) {
+ this.connections = connections;
+ }
+
+ @Override
+ public String toString() {
+ return "ConfigConnectionsBean [connections=" + ConfigRootBean.listDump(connections,1) + "]";
+ }
+
+}
diff --git a/src/main/src/conf/ConfigGlobalsBean.java b/src/main/src/conf/ConfigGlobalsBean.java
new file mode 100644
index 0000000..256acee
--- /dev/null
+++ b/src/main/src/conf/ConfigGlobalsBean.java
@@ -0,0 +1,41 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+/**
+ * Generated Configuration Bean
+ */
+public class ConfigGlobalsBean {
+ private int maxExecTime;
+
+ public int getMaxExecTime() {
+ return maxExecTime;
+ }
+
+ public void setMaxExecTime(int maxExecTime) {
+ this.maxExecTime = maxExecTime;
+ }
+
+ @Override
+ public String toString() {
+ return "ConfigGlobalsBean [maxExecTime=" + maxExecTime + "]";
+ }
+}
diff --git a/src/main/src/conf/ConfigOpLimitsBean.java b/src/main/src/conf/ConfigOpLimitsBean.java
new file mode 100644
index 0000000..8f68e8c
--- /dev/null
+++ b/src/main/src/conf/ConfigOpLimitsBean.java
@@ -0,0 +1,55 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+/**
+ * Generated Configuration Bean
+ */
+public class ConfigOpLimitsBean {
+ private int insert;
+ private int update;
+ private int delete;
+
+ public int getInsert() {
+ return insert;
+ }
+ public void setInsert(int insert) {
+ this.insert = insert;
+ }
+ public int getUpdate() {
+ return update;
+ }
+ public void setUpdate(int update) {
+ this.update = update;
+ }
+ public int getDelete() {
+ return delete;
+ }
+ public void setDelete(int delete) {
+ this.delete = delete;
+ }
+
+ @Override
+ public String toString() {
+ return "ConfigOpLimitsBean [insert=" + insert + ", update=" + update
+ + ", delete=" + delete + "]";
+ }
+}
diff --git a/src/main/src/conf/ConfigRootBean.java b/src/main/src/conf/ConfigRootBean.java
new file mode 100644
index 0000000..acbbd49
--- /dev/null
+++ b/src/main/src/conf/ConfigRootBean.java
@@ -0,0 +1,73 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+import java.util.List;
+
+/**
+ * Generated Configuration Bean
+ */
+public class ConfigRootBean {
+
+ private ConfigGlobalsBean globals;
+ private List<ConfigTaskBean> tasks;
+
+ public ConfigGlobalsBean getGlobals() {
+ return globals;
+ }
+ public void setGlobals(ConfigGlobalsBean globals) {
+ this.globals = globals;
+ }
+
+ public List<ConfigTaskBean> getTasks() {
+ return tasks;
+ }
+ public void setTasks(List<ConfigTaskBean> tasks) {
+ this.tasks = tasks;
+ }
+
+ @Override
+ public String toString() {
+ return "ConfigRootBean [globals=" + globals + ", tasks=" + listDump(tasks, 1) + "]";
+ }
+
+
+ public static <T> String listDump(List<T> list, int ident) {
+ StringBuffer buf = new StringBuffer();
+ buf.append('{');
+ for (T item : list) {
+ buf.append('\n');
+ for (int i = 0; i < ident; i++) {
+ buf.append('\t');
+ }
+ buf.append(item.toString());
+ buf.append(',');
+ }
+ buf.append('\n');
+ for (int i = 0; i < ident-1; i++) {
+ buf.append('\t');
+ }
+ buf.append('}');
+ return buf.toString();
+ }
+
+
+}
diff --git a/src/main/src/conf/ConfigSrcOrDestBean.java b/src/main/src/conf/ConfigSrcOrDestBean.java
new file mode 100644
index 0000000..5be1674
--- /dev/null
+++ b/src/main/src/conf/ConfigSrcOrDestBean.java
@@ -0,0 +1,96 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+import data.filters.MVDataCombiner;
+
+/**
+ * Generated Configuration Bean
+ */
+public class ConfigSrcOrDestBean {
+
+ public enum SourceKind { csv, ldap, sorted_csv, sql };
+
+ private String name;
+ private SourceKind kind;
+ private String conn;
+ private MVDataCombiner.MVDataCombineMode mode;
+ private String query;
+ private String path;
+ private String attr;
+ private String base;
+
+ public String getName() {
+ return name;
+ }
+ public void setName(String name) {
+ this.name = name;
+ }
+ public SourceKind getKind() {
+ return kind;
+ }
+ public void setKind(SourceKind kind) {
+ this.kind = kind;
+ }
+ public String getConn() {
+ return conn;
+ }
+ public void setConn(String conn) {
+ this.conn = conn;
+ }
+ public MVDataCombiner.MVDataCombineMode getMode() {
+ return mode;
+ }
+ public void setMode(MVDataCombiner.MVDataCombineMode mode) {
+ this.mode = mode;
+ }
+ public String getQuery() {
+ return query;
+ }
+ public void setQuery(String query) {
+ this.query = query;
+ }
+ public String getPath() {
+ return path;
+ }
+ public void setPath(String path) {
+ this.path = path;
+ }
+ public String getAttr() {
+ return attr;
+ }
+ public void setAttr(String attr) {
+ this.attr = attr;
+ }
+ public String getBase() {
+ return base;
+ }
+ public void setBase(String base) {
+ this.base = base;
+ }
+
+ @Override
+ public String toString() {
+ return "ConfigSrcOrDestBean [name=" + name + ", kind=" + kind
+ + ", conn=" + conn + ", mode=" + mode + ", query=" + query
+ + ", path=" + path + ", attr=" + attr + ", base=" + base + "]";
+ }
+}
diff --git a/src/main/src/conf/ConfigTaskBean.java b/src/main/src/conf/ConfigTaskBean.java
new file mode 100644
index 0000000..ed34eee
--- /dev/null
+++ b/src/main/src/conf/ConfigTaskBean.java
@@ -0,0 +1,80 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+import java.util.List;
+
+/**
+ * Generated Configuration Bean
+ */
+public class ConfigTaskBean {
+
+ private String name;
+ private ConfigOpLimitsBean opLimits;
+ private List<ConfigSrcOrDestBean> sources;
+ private ConfigSrcOrDestBean destination;
+ private boolean skipReadErrors;
+ private boolean skipEntryDelete;
+
+ public String getName() {
+ return name;
+ }
+ public void setName(String name) {
+ this.name = name;
+ }
+ public ConfigOpLimitsBean getOpLimits() {
+ return opLimits;
+ }
+ public void setOpLimits(ConfigOpLimitsBean opLimits) {
+ this.opLimits = opLimits;
+ }
+ public List<ConfigSrcOrDestBean> getSources() {
+ return sources;
+ }
+ public void setSources(List<ConfigSrcOrDestBean> sources) {
+ this.sources = sources;
+ }
+ public ConfigSrcOrDestBean getDestination() {
+ return destination;
+ }
+ public void setDestination(ConfigSrcOrDestBean destination) {
+ this.destination = destination;
+ }
+ public boolean isSkipReadErrors() {
+ return skipReadErrors;
+ }
+ public void setSkipReadErrors(boolean skipReadErrors) {
+ this.skipReadErrors = skipReadErrors;
+ }
+ public boolean isSkipEntryDelete() {
+ return skipEntryDelete;
+ }
+ public void setSkipEntryDelete(boolean skipDelete) {
+ this.skipEntryDelete = skipDelete;
+ }
+ @Override
+ public String toString() {
+ return "ConfigTaskBean [name=" + name + ", opLimits=" + opLimits
+ + ", sources=" + sources + ", destination=" + destination
+ + ", skipReadErrors=" + skipReadErrors + ", skipEntryDelete="
+ + skipEntryDelete + "]";
+ }
+}
diff --git a/src/main/src/conf/SSSyncConfParser.java b/src/main/src/conf/SSSyncConfParser.java
new file mode 100644
index 0000000..42dc760
--- /dev/null
+++ b/src/main/src/conf/SSSyncConfParser.java
@@ -0,0 +1,65 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.text.ParseException;
+
+import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.constructor.Constructor;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public class SSSyncConfParser {
+
+ public static ConfigRootBean loadMainConfig(String mainConfigFile) throws FileNotFoundException, ParseException {
+ Yaml yamlMain = new Yaml(new Constructor(ConfigRootBean.class));
+
+ //TODO : try to prevent weird exceptions when config is not respecting the implicit grammar of the bean tree
+
+ ConfigRootBean confMain = (ConfigRootBean) yamlMain.load(new FileInputStream(mainConfigFile));
+
+ if ( confMain == null || confMain.getGlobals() == null ) {
+ throw new ParseException("Config parser has returned a null item", 0);
+ }
+
+ // TODO : check config sanity and completeness
+
+ return confMain;
+ }
+
+ public static ConfigConnectionsBean loadConnConfig(String connConfigFile) throws FileNotFoundException, ParseException {
+ Yaml yamlConn = new Yaml(new Constructor(ConfigConnectionsBean.class));
+
+ ConfigConnectionsBean confConn = (ConfigConnectionsBean) yamlConn.load(new FileInputStream(connConfigFile));
+
+ if ( confConn == null ) {
+ throw new ParseException("Config parser has return a null item", 0);
+ }
+
+ return confConn;
+ }
+
+}
diff --git a/src/main/src/conf/SSSyncConnectionsFactory.java b/src/main/src/conf/SSSyncConnectionsFactory.java
new file mode 100644
index 0000000..e747258
--- /dev/null
+++ b/src/main/src/conf/SSSyncConnectionsFactory.java
@@ -0,0 +1,61 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+import data.io.ConnectionsHolder;
+import data.io.ldap.LDAPConnectionWrapper;
+import data.io.sql.SQLConnectionWrapper;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public class SSSyncConnectionsFactory {
+
+ /**
+ * Setup all connections described in config
+ * @return
+ * @throws Exception
+ */
+ public static ConnectionsHolder setupConnections(ConfigConnectionsBean confConn) throws Exception {
+ ConnectionsHolder connections = new ConnectionsHolder();
+
+ for ( ConfigConnectionBean conn : confConn.getConnections() ) {
+ switch (conn.getType()) {
+ case jdbc:
+ SQLConnectionWrapper connSQL = new SQLConnectionWrapper(conn.getDbms(), conn.getHost(), conn.getPort(), conn.getRess(), conn.getUser(), conn.getPass(), conn.getDb());
+ connections.putConnSQL(conn.getId(), connSQL);
+ break;
+ case ldap:
+ LDAPConnectionWrapper connLDAP = new LDAPConnectionWrapper(conn.getHost(), conn.getPort(), conn.getBind(), conn.getPass());
+ connections.putConnLDAP(conn.getId(), connLDAP);
+ break;
+ default:
+ //XXX : find better Exception type
+ throw new Exception("Bad config : conn '" + conn.getId() + "' unsupported type");
+ }
+ }
+
+ return connections;
+ }
+
+}
diff --git a/src/main/src/conf/SSSyncTasksFactory.java b/src/main/src/conf/SSSyncTasksFactory.java
new file mode 100644
index 0000000..de3e8f6
--- /dev/null
+++ b/src/main/src/conf/SSSyncTasksFactory.java
@@ -0,0 +1,147 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package conf;
+
+import java.io.File;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import sync.BasicSyncTask;
+import data.filters.MVDataCombiner;
+import data.filters.MVDataCombiner.MVDataCombineMode;
+import data.io.ConnectionsHolder;
+import data.io.MVDataReader;
+import data.io.MVDataWriter;
+import data.io.SafeDataReader;
+import data.io.csv.CSVDataReader;
+import data.io.ldap.LDAPConnectionWrapper;
+import data.io.sql.SQLConnectionWrapper;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public class SSSyncTasksFactory {
+
+ /**
+ * Build tasks objects with all needed resources from a config beans tree
+ * @param conf
+ * @return
+ * @throws Exception
+ */
+ public static List<BasicSyncTask> setupTasks(ConnectionsHolder connections, ConfigRootBean confMain) throws Exception {
+ List<BasicSyncTask> tasks = new ArrayList<BasicSyncTask>();
+
+ // For each task...
+ for ( ConfigTaskBean confTask: confMain.getTasks() ) {
+ MVDataReader srcReader=null;
+
+ // Building all sources
+
+ List<ConfigSrcOrDestBean> confSources = confTask.getSources();
+ // See if we are in multiple source situation (then MVDataCombiner) or not (then simple MVDataReader)
+ if ( confSources.size() == 0 ) {
+ throw new Exception("Bad config : task '" + confTask.getName() + "' has no defined sources");
+ } else if ( confSources.size() == 1 ) {
+ srcReader = new SafeDataReader(_makeReader(connections, confSources.get(0), confTask.getName()), confTask.isSkipReadErrors());
+ } else {
+ List<MVDataReader> readers = new ArrayList<MVDataReader>();
+ List<MVDataCombineMode> mergeModes = new ArrayList<MVDataCombineMode>();
+
+ // For each source of the future MVDataCombiner...
+ for ( ConfigSrcOrDestBean confSource: confSources ) {
+ // Create and add the reader and his parameters
+ readers.add(new SafeDataReader(_makeReader(connections, confSource, confTask.getName()), confTask.isSkipReadErrors()));
+ mergeModes.add(confSource.getMode());
+ }
+
+ srcReader = new MVDataCombiner("srcCombiner", readers.toArray(new MVDataReader[0]), mergeModes.toArray(new MVDataCombineMode[0]));
+ }
+
+ // Building destination
+
+ MVDataReader dstReader=null;
+ MVDataWriter dstWriter=null;
+
+ ConfigSrcOrDestBean confDestination = confTask.getDestination();
+ switch ( confDestination.getKind() ) {
+ case ldap:
+ LDAPConnectionWrapper builder = connections.getLDAPConnectionBuilder(confDestination.getConn());
+ // TODO : configurable lookAhead
+ MVDataReader tmpReader = builder.newFlatReader(confDestination.getName()+"_reader", confDestination.getBase(), confDestination.getAttr(), 128);
+ dstReader = new SafeDataReader(tmpReader, false);
+ dstWriter = builder.newFlatWriter(confDestination.getBase(), confDestination.getAttr());
+ break;
+ default:
+ throw new Exception("Bad config : task '" + confTask.getName() + "' unsupported destination kind");
+ }
+
+ // Then building the sync task and add it to the task list
+ int maxInserts = confTask.getOpLimits().getInsert();
+ int maxUpdates = confTask.getOpLimits().getUpdate();
+ int maxDeletes = confTask.getOpLimits().getDelete();
+
+ BasicSyncTask task = new BasicSyncTask(confTask.getName(), false, srcReader, dstReader, dstWriter);
+ task.setOperationLimits(maxInserts, maxUpdates, maxDeletes);
+ task.setSkipEntryDelete(confTask.isSkipEntryDelete());
+ tasks.add(task);
+ }
+
+ return tasks;
+ }
+
+ /**
+ * Helper function to make a new reader from an existing connection
+ * @param confSource
+ * @param taskName
+ * @return
+ * @throws Exception
+ */
+ private static MVDataReader _makeReader(ConnectionsHolder connections, ConfigSrcOrDestBean confSource, String taskName) throws Exception {
+ MVDataReader reader=null;
+ switch (confSource.getKind()) {
+ case csv:
+ reader = new CSVDataReader(confSource.getName(), new FileReader(confSource.getPath()), false);
+ break;
+ case ldap:
+ LDAPConnectionWrapper ldapConnBuilder = connections.getLDAPConnectionBuilder(confSource.getConn());
+ //FIXME : if conf error, get...ConnectionBuilder could return null
+ //TODO : configurable lookAhead
+ reader = ldapConnBuilder.newFlatReader(confSource.getName(), confSource.getBase(), confSource.getAttr(), 128);
+ break;
+ case sorted_csv:
+ reader = new CSVDataReader(confSource.getName(), new FileReader(confSource.getPath()), true);
+ break;
+ case sql:
+ SQLConnectionWrapper sqlConnBuilder = connections.getSQLConnectionBuilder(confSource.getConn());
+ //TODO We assume the query config item is a filepath. It isn't checked anywhere.
+ reader = sqlConnBuilder.newReader(confSource.getName(), new File(confSource.getQuery()));
+ break;
+ default:
+ throw new Exception("Bad config : task '" + taskName + "' unsupported source kind");
+ }
+
+ return reader;
+ }
+
+}
diff --git a/src/main/src/data/io/ConnectionsHolder.java b/src/main/src/data/io/ConnectionsHolder.java
new file mode 100644
index 0000000..3a6e527
--- /dev/null
+++ b/src/main/src/data/io/ConnectionsHolder.java
@@ -0,0 +1,81 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.HashMap;
+
+import data.io.ldap.LDAPConnectionWrapper;
+import data.io.sql.SQLConnectionWrapper;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public class ConnectionsHolder implements Closeable {
+
+ public final HashMap<String, LDAPConnectionWrapper> connMapLDAP;
+ public final HashMap<String, SQLConnectionWrapper> connMapSQL;
+
+ //TODO : with some refactoring, this class may disappear
+ /**
+ * Bean class to keep track of all opened connections in a single object
+ */
+ public ConnectionsHolder() {
+ this.connMapLDAP = new HashMap<String, LDAPConnectionWrapper>();
+ this.connMapSQL = new HashMap<String, SQLConnectionWrapper>();
+ }
+
+ public LDAPConnectionWrapper getLDAPConnectionBuilder(String conn) {
+ return connMapLDAP.get(conn);
+ }
+
+ public SQLConnectionWrapper getSQLConnectionBuilder(String conn) {
+ return connMapSQL.get(conn);
+ }
+
+ public void putConnLDAP(String connId, LDAPConnectionWrapper connLDAP) {
+ this.connMapLDAP.put(connId, connLDAP);
+ }
+
+ public void putConnSQL(String connId, SQLConnectionWrapper connSQL) {
+ this.connMapSQL.put(connId, connSQL);
+ }
+
+ /**
+ * Close all connections
+ */
+ @Override
+ public void close() throws IOException {
+ // XXX : this will stop at first uncloseable connection. It isn't a very interesting problem however.
+ for ( LDAPConnectionWrapper connLDAP: connMapLDAP.values() ) {
+ connLDAP.close();
+ }
+ for ( SQLConnectionWrapper connSQL: connMapSQL.values() ) {
+ connSQL.close();
+ }
+ }
+
+
+
+}
diff --git a/src/main/src/data/io/SafeDataReader.java b/src/main/src/data/io/SafeDataReader.java
new file mode 100644
index 0000000..2c5dda9
--- /dev/null
+++ b/src/main/src/data/io/SafeDataReader.java
@@ -0,0 +1,155 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package data.io;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.log4j.Logger;
+
+import data.MVDataEntry;
+
+/**
+ * Multi-valued "safe" stream reader proxy.
+ * Adds logging and skipReadError mode feature. Check if items are well ordered.
+ * Ensures consistency of hasNext() / next() even if source stream is faulty.
+ * Never returns null items but throw NoSuchElementException if no other choices.
+ *
+ * @author lpouzenc
+ */
+public class SafeDataReader extends AbstractMVDataReader {
+
+ private static final Logger logger = Logger.getLogger(SafeDataReader.class.getName());
+
+ private final MVDataReader src;
+ /**
+ * If true, continue even in case of read errors
+ */
+ private final boolean skipReadErrors;
+
+ private transient Iterator<MVDataEntry> srcIt;
+ private transient boolean abort;
+ private transient MVDataEntry previousData;
+
+
+ public SafeDataReader(MVDataReader src, boolean skipReadErrors) {
+ this.src = src;
+ this.dataSourceName = src.getDataSourceName();
+ this.skipReadErrors = skipReadErrors;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public Iterator<MVDataEntry> iterator() {
+ // Reset everything
+ srcIt = src.iterator();
+ abort = false;
+ previousData = null;
+
+ return this;
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public boolean hasNext() {
+ return (!abort && srcIt.hasNext());
+ }
+
+ /**
+ * {@inheritDoc}
+ */
+ @Override
+ public MVDataEntry next() {
+ boolean alreadyWarned=false;
+ boolean done=false;
+ MVDataEntry entry = null;
+
+ // Prepare an hint for read exception (knowledge of last successfully read entry could help)
+ String hint = ( previousData != null )?previousData.getKey():"(nothing)";
+
+ // Seek for the next valid entry
+ while (!this.abort && !done && srcIt.hasNext()) {
+
+ // Try to read next entry
+ try {
+ entry=src.next();
+ if ( entry == null ) throw new NoSuchElementException("Null item returned");
+ } catch (Exception e) {
+ logger.warn(src.getDataSourceName() + " : exception when seeking next valid entry after " + hint, e);
+ entry = null; // Make sure don't re-use a previous entry
+ }
+
+ // Sanity checks
+ boolean valid = ( entry != null && entry.isValid() );
+ //XXX Regex should be a parameter
+ if ( valid && !entry.getKey().matches("^\\p{Print}+$") ) {
+ logger.warn(src.getDataSourceName() + " : Invalid key found : '" + entry.getKey().replaceAll("[^\\p{Print}]", "?") + "' after " + hint);
+ valid = false;
+ }
+
+
+ // Two branches : If valid, check ordering then skip or done. If invalid : skip or abort.
+ if ( valid ) {
+ // Ensure that data.key is greater than previousData.key or abort
+ if ( previousData != null && entry.getKey().compareTo(previousData.getKey()) <= 0 ) {
+ //TODO : this is almost useless in case of reverse-sortered query because everything will be deleted by the Syncer before asking the second item
+ logger.error(src.getDataSourceName() + " : Input data is not well ordered but the sync task require it : '"
+ + entry.getKey() + "' is not lexicographically greater than '" + previousData.getKey() + "'");
+ // Escape the while loop
+ abort=true; continue;
+ }
+
+ // We have found a valid entry, so escape gracefully the loop
+ done=true;
+ } else {
+ // Log read problems and choose between skip or abort
+ if ( ! this.skipReadErrors ) {
+ logger.error(src.getDataSourceName() + " has returned an invalid entry after " + hint);
+ // Escape the while loop
+ abort=true; continue;
+ }
+ if ( !alreadyWarned ) {
+ alreadyWarned=true;
+ logger.info("Invalid entry read but skipReadErrors is enabled, will try to read next entry (warned only once)");
+ }
+
+ // We don't have a valid entry, give a chance to the next iteration
+ done=false;
+ } /* if ( valid )*/
+
+ } /* while */
+
+ // If we don't have found anything valid, throw exception (better semantics than returning null)
+ if (!done) {
+ throw new NoSuchElementException();
+ }
+
+ // Keep track of previous read record
+ // -> for hinting in log messages when bad things happens
+ // -> to check if entries are well ordered
+ previousData=entry;
+ return entry;
+ }
+}
diff --git a/src/main/src/sync/BasicSyncTask.java b/src/main/src/sync/BasicSyncTask.java
new file mode 100644
index 0000000..24f34a8
--- /dev/null
+++ b/src/main/src/sync/BasicSyncTask.java
@@ -0,0 +1,292 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package sync;
+
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+
+import data.MVDataEntry;
+import data.io.MVDataReader;
+import data.io.MVDataWriter;
+
+/**
+ * Basic one-way synchronization code. Uses MVDataEntry semantics.
+ * Each entry has a key and a set of multi-valued attributes, like LDAP entries.
+ * Data source is a MVDataReader. Multiple source could be used via MVDataCombiner.
+ * <br/><br/>
+ * <b>Warnings :</b> needs MVDataReaders that give key-sorted results. This sync will try
+ * to delete entries that exists on destination side and don't exist at source side.
+ * Extra attributes in existing entries on destination side are preserved.
+ * Look like useful for account's failure password count for instance.
+ * <br/><br/>
+ * <b>Notes :</b> Null value and empty strings are not allowed in MVDataEntry, so they are not sync'ed.
+ *
+ * @author lpouzenc
+ */
+public class BasicSyncTask extends AbstractSyncTask {
+ private static final Logger logger = Logger.getLogger(BasicSyncTask.class.getName());
+
+ /**
+ * Source data stream (read-only)
+ */
+ private final MVDataReader srcReader;
+ /**
+ * Destination data stream (read)
+ */
+ private final MVDataReader dstReader;
+ /**
+ * Destination data stream (write)
+ */
+ private final MVDataWriter dstWriter;
+
+ /**
+ * If true, disable removal of data on destination side even if detected as obsolete
+ */
+ private boolean skipEntryDelete;
+
+
+ private int maxInserts;
+ private int maxUpdates;
+ private int maxDeletes;
+
+ private transient int curInserts;
+ private transient int curUpdates;
+ private transient int curDeletes;
+
+
+ /**
+ * BasicSyncTask constructor
+ * Assumes that the *Readers have iterators that returns entries sorted by lexicographical ascending key
+ * @param taskName Friendly name of this task (for tracing in log files)
+ * @param srcReader Source data stream (read-only)
+ * @param dstReader Destination data stream (read)
+ * @param dstWriter Destination data stream (write)
+ */
+ public BasicSyncTask(String taskName, boolean skipDelete, MVDataReader srcReader, MVDataReader dstReader, MVDataWriter dstWriter) {
+ this.taskName = taskName;
+ this.srcReader = srcReader;
+ this.dstReader = dstReader;
+ this.dstWriter = dstWriter;
+
+ this.maxInserts = 0;
+ this.maxUpdates = 0;
+ this.maxDeletes = 0;
+ }
+
+ public Boolean call() {
+ logger.info("task " + taskName + " : starting " + (dryRun?"dry-run":"real") + " pass");
+ // Better stack traces "call()" don't say "what"
+ boolean success = syncTaskRun();
+ logger.info("task " + taskName + " : " + (success?"terminated successfully":"aborted"));
+
+ return success;
+ }
+
+ private boolean syncTaskRun() {
+ curInserts=0;
+ curUpdates=0;
+ curDeletes=0;
+ dstWriter.setDryRun(dryRun);
+
+ Iterator<MVDataEntry> itSrc = srcReader.iterator();
+ Iterator<MVDataEntry> itDst = dstReader.iterator();
+ MVDataEntry src = null, dst = null;
+ boolean srcExhausted = false;
+ boolean dstExhausted = false;
+ boolean abort = false;
+ boolean done = false;
+ while ( !abort && !done ) {
+
+ // Look-ahead srcReader if previous has been "poped" (or if never read yet)
+ if ( src == null ) {
+ if ( !srcExhausted ) {
+ srcExhausted = !itSrc.hasNext();
+ }
+ if ( !srcExhausted ) {
+ try {
+ src=itSrc.next();
+ logger.trace("src read : " + src);
+ } catch (Exception e) {
+ logger.error("Read failure detected on " + srcReader.getDataSourceName() + ". Aborting.", e);
+ // Escape from the while loop
+ abort=true; continue;
+ }
+ }
+ }
+
+ // Look-ahead dstReader if previous has been "poped" (or if never read yet)
+ if ( dst == null ) {
+ if ( !dstExhausted ) {
+ dstExhausted = !itDst.hasNext();
+ }
+ if ( !dstExhausted ) {
+ try {
+ dst = itDst.next();
+ logger.trace("dst read : " + dst);
+ } catch (NoSuchElementException e) {
+ logger.error("Read failure detected on " + dstReader.getDataSourceName() + ". Aborting.", e);
+ // Escape from the while loop
+ abort=true; continue;
+ }
+ }
+ }
+
+ // Error-free cases (no problems while reading data)
+ int compare;
+ if ( !srcExhausted && !dstExhausted ) {
+ // General case : check order precedence to take an action
+ compare = src.compareTo(dst);
+ } else if ( !srcExhausted && dstExhausted ) {
+ // Particular case : dst is exhausted, it's like ( src < dst )
+ compare=-1;
+ } else if ( srcExhausted && !dstExhausted ) {
+ // Particular case : src is exhausted, it's like ( src > dst )
+ compare=1;
+ } else /* ( srcExhausted && dstExhausted ) */ {
+ // Particular case : everything is synchronized
+ // Exit gracefully the while loop
+ done=true; continue;
+ }
+
+ logger.trace("compare : " + compare);
+
+ boolean actionRealized = false;
+ // Take an action (insert/update/delete)
+ if ( compare < 0 ) {
+ actionRealized = _insert(src);
+ src = null;
+ // preserve dst until src key is not greater
+ } else if ( compare > 0 ) {
+ // dst current entry doesn't exists anymore (src key is greater than dst key)
+ actionRealized = _delete(dst);
+ // preserve src until dst key is not greater
+ dst = null;
+ } else /* ( compare == 0 ) */ {
+ // src current entry already exists in dst, update it if necessary
+ Set<String> changedAttr = src.getChangedAttributes(dst);
+ if ( ! changedAttr.isEmpty() ) {
+ actionRealized = _update(src,dst,changedAttr);
+ } else {
+ // Already up-to-date, nothing to do
+ actionRealized = true;
+ }
+ // Both src and dst have been used
+ src = null;
+ dst = null;
+ }
+ abort = !actionRealized;
+ } /* while */
+
+ return !abort;
+ } /* _taskRunSync() */
+
+ private boolean _insert(MVDataEntry entry) {
+
+ if ( maxInserts > 0 && curInserts >= maxInserts ) {
+ logger.error("Max insert limit reached (" + maxInserts + ")" );
+ return false;
+ }
+
+ logger.debug("dstWriter : Action\n-> Insert " + entry);
+ try {
+ dstWriter.insert(entry);
+ } catch (Exception e) {
+ logger.error("Exception occured while inserting", e);
+ return false;
+ }
+
+ curInserts++;
+ return true;
+ }
+
+ private boolean _update(MVDataEntry updatedEntry, MVDataEntry originalEntry, Set<String> attrToUpdate) {
+ if ( maxUpdates > 0 && curUpdates >= maxUpdates ) {
+ logger.error("Max update limit reached (" + maxUpdates + ")");
+ return false;
+ }
+
+ logger.debug("dstWriter : Action\n-> Update " + updatedEntry + "\n-> changed attributes : " + attrToUpdate);
+ try {
+ dstWriter.update(updatedEntry, originalEntry, attrToUpdate);
+ } catch (Exception e) {
+ logger.error("Exception occured while updating", e);
+ return false;
+ }
+
+ curUpdates++;
+ return true;
+ }
+
+ private boolean _delete(MVDataEntry entry) {
+ if ( skipEntryDelete ) {
+ logger.info("dstWriter : skipping deletion for key " + entry.getKey());
+ return true;
+ }
+
+ if ( maxDeletes > 0 && curDeletes >= maxDeletes ) {
+ logger.error("Max delete limit reached (" + maxDeletes + ")");
+ return false;
+ }
+ logger.debug("dstWriter : Action\n-> Delete " + entry);
+ try {
+ dstWriter.delete(entry);
+ } catch (Exception e) {
+ logger.error("Exception occured while deleting", e);
+ return false;
+ }
+
+ curDeletes++;
+ return true;
+ }
+
+ // Boring accessors
+
+ /**
+ * Setter to fix limits about operations counts (safeguard)
+ * @param maxInserts
+ * @param maxUpdates
+ * @param maxDeletes
+ */
+ public void setOperationLimits(int maxInserts, int maxUpdates, int maxDeletes) {
+ this.maxInserts = maxInserts;
+ this.maxUpdates = maxUpdates;
+ this.maxDeletes = maxDeletes;
+ }
+
+ /**
+ * @return the skipEntryDelete
+ */
+ public boolean isSkipEntryDelete() {
+ return skipEntryDelete;
+ }
+
+ /**
+ * @param skipEntryDelete the skipEntryDelete to set
+ */
+ public void setSkipEntryDelete(boolean skipEntryDelete) {
+ this.skipEntryDelete = skipEntryDelete;
+ }
+
+}
diff --git a/src/main/src/utils/JVMStatsDumper.java b/src/main/src/utils/JVMStatsDumper.java
new file mode 100644
index 0000000..41f1d97
--- /dev/null
+++ b/src/main/src/utils/JVMStatsDumper.java
@@ -0,0 +1,111 @@
+/*
+ * SSSync, a Simple and Stupid Synchronizer for data with multi-valued attributes
+ * Copyright (C) 2014 Ludovic Pouzenc <ludovic@pouzenc.fr>
+ *
+ * This file is part of SSSync.
+ *
+ * SSSync is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * SSSync is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with SSSync. If not, see <http://www.gnu.org/licenses/>
+ */
+
+package utils;
+
+import java.lang.management.GarbageCollectorMXBean;
+import java.lang.management.ManagementFactory;
+import java.lang.management.MemoryPoolMXBean;
+import java.lang.management.MemoryUsage;
+import java.lang.management.RuntimeMXBean;
+
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+/**
+ * TODO javadoc
+ *
+ * @author lpouzenc
+ */
+public class JVMStatsDumper {
+ private static final Logger logger = Logger.getLogger(JVMStatsDumper.class.getName());
+
+ public static void logGCStats() {
+ // Skip all string construction if will not print this stuff
+ if ( logger.getLevel().isGreaterOrEqual(Level.INFO) ) { return; }
+
+ long totalGarbageCollections = 0;
+ long garbageCollectionTime = 0;
+
+ final String gcDumpHeader="Dumping Garbage Collector statistics\n" +
+ "+--------------------+-----------------------------+\n" +
+ "+ GC Name + Count + Time (ms) +\n" +
+ "+--------------------+--------------+--------------+\n";
+
+ StringBuilder sb = new StringBuilder(1024);
+ sb.append(gcDumpHeader);
+
+ for(GarbageCollectorMXBean gc : ManagementFactory.getGarbageCollectorMXBeans()) {
+
+ long count = gc.getCollectionCount();
+ long time = gc.getCollectionTime();
+
+ sb.append(String.format("+ %18s + %,12d + %,12d +%n", gc.getName(), count, time));
+
+ if(count >= 0) totalGarbageCollections += count;
+ if(time >= 0) garbageCollectionTime += time;
+ }
+
+ sb.append("+ + + +\n");
+ sb.append(String.format("+ %18s + %,12d + %,12d +%n",
+ "Total", totalGarbageCollections, garbageCollectionTime
+ ));
+ sb.append("+--------------------+--------------+--------------+\n");
+
+ sb.append("JVM arguments : ");
+ RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean();
+ for ( String arg : runtimeMxBean.getInputArguments() ) {
+ sb.append(arg + " ");
+ }
+
+ logger.debug(sb);
+ }
+
+ /**
+ * Helper function to log the current memory usage
+ */
+ public static void logMemoryUsage() {
+ // Skip all string construction if will not print this stuff
+ if ( logger.getLevel().isGreaterOrEqual(Level.INFO) ) { return; }
+
+ final String memDumpHeader="Dumping memory statistics\n" +
+ "+--------------------------------------------------------------------------------+\n" +
+ "+ + Current (kio) + Peak (kio) +\n" +
+ "+ Pool +-----------------------------------------------------------+\n" +
+ "+ + Used + Committed + Used + Committed +\n" +
+ "+--------------------+--------------+--------------+--------------+--------------+\n";
+
+ StringBuilder sb = new StringBuilder(1024);
+ sb.append(memDumpHeader);
+
+ for (MemoryPoolMXBean pool : ManagementFactory.getMemoryPoolMXBeans()) {
+ MemoryUsage peak = pool.getPeakUsage();
+ MemoryUsage curr = pool.getUsage();
+ sb.append(String.format("+ %18s + %,12d + %,12d + %,12d + %,12d +%n",
+ pool.getName(),curr.getUsed()/1024, curr.getCommitted()/1024, peak.getUsed()/1024, peak.getCommitted()/1024
+ ));
+ pool.resetPeakUsage(); //XXX Maybe this is not a global action and is useless on a temporary object used once
+ }
+ sb.append("+--------------------+--------------+--------------+--------------+--------------+\n");
+
+ logger.debug(sb);
+ }
+
+}
diff --git a/src/main/sssync.sh b/src/main/sssync.sh
new file mode 100755
index 0000000..43a1810
--- /dev/null
+++ b/src/main/sssync.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+cd $(dirname $0)
+java -jar SSSync.jar
diff --git a/src/sloc.sh b/src/sloc.sh
new file mode 100755
index 0000000..6b5cb02
--- /dev/null
+++ b/src/sloc.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+mkdir /tmp/sloc/{,src,JUTests}
+find */src -name *.java \! -name *Bean.java -print0 | xargs -0r cp -vt /tmp/sloc/src
+find */JUTests -name *.java \! -name *Bean.java -print0 | xargs -0r cp -vt /tmp/sloc/JUTests
+sloccount /tmp/sloc/*
+rm -r /tmp/sloc