reproducible builds with Gradle

apps/addressbook/build.gradle

@@ -1,3 +1,5 @@
+apply plugin:'war'
+
 sourceSets {
     main {
         java {
@@ -7,6 +9,14 @@ sourceSets {
 }
 
 dependencies {
-    compile project(':core')
-    compile project(':apps:jetty')
+    providedCompile project(':core')
+    providedCompile project(':apps:jetty')
+}
+
+war {
+    rootSpec.exclude('**/*.class')
+}
+
+artifacts {
+    archives war
 }

apps/i2pcontrol/build.gradle

@@ -20,3 +20,7 @@ war {
     archiveName 'jsonrpc.war'
     webXml = file('web.xml')
 }
+
+artifacts {
+    archives war
+}

apps/i2psnark/build.gradle

@@ -32,7 +32,7 @@ task i2psnarkJar(type: Jar) {
 // TODO: standalone jar.  This is rather involved!
 
 artifacts {
-    archives i2psnarkJar
+    archives i2psnarkJar,war
 }
 
 // Create the java files from the po files. The jar task will compile them.

apps/i2ptunnel/build.gradle

@@ -22,6 +22,9 @@ dependencies {
     providedCompile project(':apps:ministreaming')
     compile 'gnu.getopt:java-getopt:1.0.13'
     providedCompile project(':apps:jetty')
+
+    implementation 'org.apache.ant:ant:1.10.10'
+    implementation fileTree("../jetty/apache-tomacat-${tomcatVersion}")
 }
 
 // Create the java files from the po files. The jar task will compile them.
@@ -62,8 +65,73 @@ task i2ptunnelJar(type: Jar) {
         into "net/i2p/i2ptunnel/resources"
     })
 }
+
+task helpersJar(type: Jar) {
+    from sourceSets.main.output
+    include '**/EditBean.class'
+    include '**/IndexBean.class'
+    include 'net/i2p/i2ptunnel/ui/**'
+    include 'net/i2p/i2ptunnel/web/SSLHelper.class'
+    archiveBaseName='i2pTunnelHelpers'
+}
+
+task precompileJsp(type : JavaExec) {
+    classpath = sourceSets.main.runtimeClasspath
+    main = 'net.i2p.servlet.util.JspC'
+
+    jvmArgs "-Dtomcat.util.scan.StandardJarScanFilter.jarsToSkip=commons-collections.jar,junit.jar,junit4.jar"
+    jvmArgs "-Dbuild.reproducible=true"
+
+    args "-d"
+    args "jsp/WEB-INF/classes"
+    args "-v"
+    args "-p"
+    args "net.i2p.i2ptunnel.jsp"
+    args "-webinc"
+    args "$buildDir/web-fragment.xml"
+    args "-webapp"
+    args "jsp"
+    doLast {
+        def output = new File("$buildDir/compiledJsps")
+        output.mkdirs()
+        ant.javac(srcDir: "jsp/WEB-INF/classes",
+                    classpath: sourceSets.main.runtimeClasspath.asPath,
+                    debug : true,
+                    includeAntRuntime : false,
+                    deprecation : "on",
+                    source: project.sourceCompatibility,
+                    target: project.targetCompatibility,
+                    destDir:file("$buildDir/compiledJsps"))
+
+        def fragment = file("$buildDir/web-fragment.xml").text
+        def templateXML = file("jsp/web.xml").text
+        def webXML = templateXML.replace("<!-- precompiled servlets -->", fragment)
+
+        def multipart = "<multipart-config>" +
+                        "<max-file-size>134217728</max-file-size>" +
+                        "<max-request-size>134217728</max-request-size>" +
+                        "<file-size-threshold>262144</file-size-threshold>" +
+                        "</multipart-config>"
+
+        def multipartServlets = ["register"]
+                                
+
+        multipartServlets = multipartServlets.collect {
+            "<servlet-class>net.i2p.i2ptunnel.jsp.${it}_jsp</servlet-class>"
+        }
+
+        multipartServlets.each {
+            webXML = webXML.replace(it, it + multipart)
+        }
+
+        file("$buildDir/web.xml").text = webXML
+    }
+
+
+}
+
 i2ptunnelJar.dependsOn bundleProxy
-war.dependsOn bundle
+war.dependsOn bundle,precompileJsp
 
 // not needed unless we're building for both android and regular
 task tempBeansJar(type: Jar) {
@@ -85,7 +153,7 @@ task uiJar(type: Jar) {
 }
 
 artifacts {
-    archives i2ptunnelJar   //, tempBeansJar, uiJar
+    archives i2ptunnelJar,war   //, tempBeansJar, uiJar
 }
 
 war {
@@ -93,6 +161,11 @@ war {
     include '**/ui/*.class'
     include '**/IndexBean.class'
     from 'jsp'
+    from ("$buildDir/compiledJsps") {
+        include '**/*.class'
+        into "WEB-INF/classes"
+    }
+    rootSpec.exclude('**/*.jar')
     rootSpec.exclude('/net/i2p/i2ptunnel/*.class')
     rootSpec.exclude('/net/i2p/i2ptunnel/access')
     rootSpec.exclude('/net/i2p/i2ptunnel/irc')
@@ -102,9 +175,11 @@ war {
     rootSpec.exclude('/net/i2p/i2ptunnel/streamr')
     rootSpec.exclude('/net/i2p/i2ptunnel/udp')
     rootSpec.exclude('/net/i2p/i2ptunnel/udpTunnel')
-    exclude 'jsp/web.xml'
-    exclude '*.jsi'
-    exclude '*.jsp'
-    webXml = file('jsp/web.xml')
+    from("jsp") {
+        exclude 'web.xml'
+        exclude '*.jsi'
+        exclude '*.jsp'
+    }
+    webXml = file("$buildDir/web.xml")
 }
 

apps/imagegen/build.gradle

@@ -14,7 +14,7 @@ sourceSets {
 }
 
 dependencies {
-    compile project(':core')
+    providedCompile project(':core')
     providedCompile project(':apps:jetty')
 }
 
@@ -23,3 +23,7 @@ war {
     from 'imagegen/webapp/src/main/webapp/index.html'
     webXml = file('imagegen/webapp/src/main/webapp/WEB-INF/web.xml')
 }
+
+artifacts {
+    archives war
+}

apps/jetty/build.gradle

@@ -10,7 +10,7 @@ sourceSets {
 
 dependencies {
     ext.jettyVersion = '9.3.29.v20201019'
-    ext.tomcatVersion = '9.0.40'
+    ext.tomcatVersion = "${tomcatVersion}"
     compile project(':core')
     compile 'org.eclipse.jetty:jetty-http:' + ext.jettyVersion
     compile 'org.eclipse.jetty:jetty-io:' + ext.jettyVersion

apps/routerconsole/build.gradle

@@ -18,11 +18,15 @@ dependencies {
     compile project(':core')
     compile project(':router')
     compile project(':installer')
+
+    // below were providedCompile
     providedCompile project(':apps:desktopgui')
     providedCompile project(':apps:systray')
     providedCompile project(':apps:jetty')
     providedCompile project(':apps:jrobin')
 
+    implementation 'org.apache.ant:ant:1.10.10'
+    implementation fileTree("../jetty/apache-tomacat-${tomcatVersion}")
     testCompile "org.scala-lang:scala-library:2.12.4"
     testCompile 'org.scalatest:scalatest_2.12:3.0.4'
 }
@@ -77,26 +81,96 @@ task consoleJar(type: Jar) {
         into "net/i2p/router/news/resources"
     })
 }
-consoleJar.dependsOn bundleJar
-war.dependsOn bundle
 
-artifacts {
-    archives consoleJar
+task helpersJar(type: Jar) {
+    from sourceSets.main.output
+    include 'net/i2p/router/web/helpers/**'
+    archiveBaseName = 'consoleHelpers'
 }
 
+
+task precompileJsp(type: JavaExec) {
+    classpath = sourceSets.main.runtimeClasspath
+    main = 'net.i2p.servlet.util.JspC'
+
+    jvmArgs "-Dtomcat.util.scan.StandardJarScanFilter.jarsToSkip=commons-collections.jar,junit.jar,junit4.jar"
+    jvmArgs "-Dbuild.reproducible=true"
+    
+    args "-d"
+    args "jsp/WEB-INF/classes"
+    args "-v"
+    args "-p"
+    args "net.i2p.router.web.jsp"
+    args "-webinc"
+    args "$buildDir/web-fragment.xml"
+    args "-webapp"
+    args "jsp"
+    doLast {
+        def output = new File("$buildDir/compiledJsps")
+        output.mkdirs()
+        ant.javac(srcDir: "jsp/WEB-INF/classes",
+                    classpath: sourceSets.main.runtimeClasspath.asPath,
+                    debug : true,
+                    includeAntRuntime : false,
+                    deprecation : "on",
+                    source: project.sourceCompatibility,
+                    target: project.targetCompatibility,
+                    destDir:file("$buildDir/compiledJsps"))
+        
+        def fragment = file("$buildDir/web-fragment.xml").text
+        def templateXML = file("jsp/web.xml").text
+        def webXML = templateXML.replace("<!-- precompiled servlets -->", fragment)
+
+        def multipart = "<multipart-config>" +
+                        "<max-file-size>134217728</max-file-size>" +
+                        "<max-request-size>134217728</max-request-size>" +
+                        "<file-size-threshold>262144</file-size-threshold>" +
+                        "</multipart-config>"
+
+        def multipartServlets = ["configplugins",
+                                "configfamily",
+                                "configreseed"]
+
+        multipartServlets = multipartServlets.collect {
+            "<servlet-class>net.i2p.router.web.jsp.${it}_jsp</servlet-class>"
+        }
+
+        multipartServlets.each {
+            webXML = webXML.replace(it, it + multipart)
+        }
+
+        file("$buildDir/web.xml").text = webXML
+    }
+        
+}
+
+consoleJar.dependsOn bundleJar
+war.dependsOn(bundle,precompileJsp)
+
+
 war {
+    rootSpec.exclude('**/*.jar')
     rootSpec.exclude('/com/vuze/**/*')
     rootSpec.exclude('/edu/internet2/**/*')
     rootSpec.exclude('/net/i2p/router/news/*')
     rootSpec.exclude('/net/i2p/router/sybil/*')
     rootSpec.exclude('/net/i2p/router/update/*')
     rootSpec.exclude('/net/i2p/router/web/*.class')
-    from 'jsp'
-    exclude 'jsp/web.xml'
-    exclude '*.jsi'
-    exclude '*.jsp'
-    webXml = file('jsp/web.xml')
+    from ("$buildDir/compiledJsps") {
+        into "WEB-INF/classes"
+    }
+    from ('jsp') {
+        exclude 'web.xml'
+        exclude '*.jsi'
+        exclude '*.jsp'
+    }
+    webXml = file("$buildDir/web.xml")
     from ('resources', {
         into "WEB-INF/classes/net/i2p/router/web/resources"
     })
 }
+
+artifacts {
+    archives war,consoleJar
+}
+

apps/susidns/build.gradle

@@ -12,8 +12,11 @@ sourceSets {
 }
 
 dependencies {
-    compile project(':core')
+    providedCompile project(':core')
     providedCompile project(':apps:jetty')
+
+    implementation 'org.apache.ant:ant:1.10.10'
+    implementation fileTree("../jetty/apache-tomacat-${tomcatVersion}")
 }
 
 // Create the java files from the po files. The jar task will compile them.
@@ -25,9 +28,84 @@ task bundle {
             println "apps/susidns/src/bundle-messages.sh".execute().text
     }
 }
-war.dependsOn bundle
+
+task precompileJsp(type: JavaExec) {
+    doFirst {
+        file("$buildDir/tmp_jsp").mkdirs()
+    }
+
+    classpath = sourceSets.main.runtimeClasspath + fileTree("src/lib")
+    main = 'net.i2p.servlet.util.JspC'
+
+    jvmArgs "-Dtomcat.util.scan.StandardJarScanFilter.jarsToSkip=commons-collections.jar,junit.jar,junit4.jar"
+    jvmArgs "-Dbuild.reproducible=true"
+
+    args "-d"
+    args "$buildDir/tmp_jsp"
+    args "-v"
+    args "-p"
+    args "i2p.susi.dns.jsp"
+    args "-webinc"
+    args "$buildDir/web-fragment.xml"
+    args "-webapp"
+    args "src/jsp"
+    doLast {
+        // normalize the order of the _jspx_dependents
+        ant.replaceregexp(file : "$buildDir/tmp_jsp/i2p/susi/dns/jsp/addressbook_jsp.java",
+                            match : "_jspx_dependants.put\\(.*\\);",
+                            replace : "//_jspx_dependants.put(@@@);",
+                            flags : "g")
+        ant.replaceregexp(file : "$buildDir/tmp_jsp/i2p/susi/dns/jsp/addressbook_jsp.java",
+                            match : "//_jspx_dependants.put(@@@);",
+                            replace : "_jspx_dependants.put("jar:file:lib/standard.jar!/META-INF/c.tld", Long.valueOf(1200000000000L));")
+        ant.replaceregexp(file : "$buildDir/tmp_jsp/i2p/susi/dns/jsp/addressbook_jsp.java",
+                            match : "//_jspx_dependants.put(@@@);",
+                            replace : "_jspx_dependants.put("file:lib/standard.jar", Long.valueOf(1200000000000L));")
+
+        def output = new File("$buildDir/compiledJsps")
+        output.mkdirs()
+
+        def classpath = sourceSets.main.runtimeClasspath + fileTree("src/lib")
+
+        ant.javac(srcDir: "$buildDir/tmp_jsp",
+                    classpath: classpath.asPath,
+                    debug : true,
+                    includeAntRuntime : false,
+                    deprecation : "on",
+                    source: project.sourceCompatibility,
+                    target: project.targetCompatibility,
+                    destDir:file("$buildDir/compiledJsps"))
+
+        def fragment = file("$buildDir/web-fragment.xml").text
+        def templateXML = file("src/WEB-INF/web-template.xml").text
+        def webXML = templateXML.replace("<!-- precompiled servlets -->", fragment)
+
+        def multipart = "<multipart-config>" +
+                        "<max-file-size>134217728</max-file-size>" +
+                        "<max-request-size>134217728</max-request-size>" +
+                        "<file-size-threshold>262144</file-size-threshold>" +
+                        "</multipart-config>"
+
+        def multipartServlets = ["addressbook"]
+
+        multipartServlets = multipartServlets.collect {
+            "<servlet-class>net.i2p.router.web.jsp.${it}_jsp</servlet-class>"
+        }
+
+        multipartServlets.each {
+            webXML = webXML.replace(it, it + multipart)
+        }
+
+        file("$buildDir/web.xml").text = webXML
+
+
+    }
+}
+
+war.dependsOn bundle,precompileJsp
 
 war {
+    rootSpec.exclude("**/*.jar")
     from 'src/jsp'
     from 'src/index.html'
     from ('src/js', {
@@ -39,7 +117,16 @@ war {
     from ('src/themes', {
         into "themes"
     })
+    from ("$buildDir/compiledJsps") {
+        include '**/*.class'
+        into 'WEB-INF/classes'
+    }
     exclude '*.jsi'
     exclude '*.jsp'
-    webXml = file('src/WEB-INF/web-template.xml')
+    webXml = file("$buildDir/web.xml")
 }
+
+artifacts {
+    archives war
+}
+

apps/susimail/build.gradle

@@ -39,3 +39,7 @@ war {
     }
     webXml = file('src/WEB-INF/web.xml')
 }
+
+artifacts {
+    archives war
+}

apps/systray/build.gradle

@@ -8,5 +8,4 @@ sourceSets {
 
 dependencies {
     compile project(':core')
-    compile files('java/lib/systray4j.jar')
 }

build.gradle

@@ -53,6 +53,7 @@ String getWorkspaceVersion() {
     "git" // TODO: extract revision
 }
 
+
 def releaseVersion = getReleaseVersion()
 def buildVersion = getBuildVersion()
 def buildExtra = getBuildExtra()
@@ -75,6 +76,8 @@ subprojects {
         testCompile 'org.mockito:mockito-core:2.5.0'
     }
 
+
+
     sourceCompatibility = 1.8
     targetCompatibility = 1.8
 
@@ -100,6 +103,9 @@ subprojects {
     tasks.withType(AbstractArchiveTask) {
         preserveFileTimestamps = false
         reproducibleFileOrder = true
+        doLast {
+            stripJar(it.archivePath)
+        }
     }
 }
 
@@ -125,3 +131,23 @@ task codeCoverageReport(type: JacocoReport) {
 }
 
 apply from: file('gradle/update.gradle')
+
+import java.util.jar.*
+void stripJar(File file) {
+    if (file.getName().endsWith('.tar'))
+        return
+    println "stripping $file"
+    File newFile = new File(file.parent, 'tmp-' + file.name)
+    newFile.withOutputStream { fout ->
+        JarOutputStream out = new JarOutputStream(fout)
+        JarFile jf = new JarFile(file)
+        jf.entries().unique {it.name}.sort {it.name}.each {
+            def copy = new JarEntry(it.name)
+            copy.time = 1001
+            out.putNextEntry(copy)
+            out << jf.getInputStream(it)
+        }
+        out.finish()
+    }
+    newFile.renameTo file
+}

gradle.properties

@@ -1,2 +1,3 @@
 # Override these in ~/.gradle/gradle.properties if necessary
 java7BootClasspath=
+tomcatVersion=9.0.45

reproducible/build.gradle

@@ -0,0 +1,93 @@
+apply plugin: 'application'
+
+application {
+    mainClassName='net.i2p.router.RouterLaunch'
+    applicationName='i2p'
+}
+
+dependencies {
+    implementation project(':router')
+    implementation project(':apps:ministreaming')
+    implementation project(':apps:streaming')
+    implementation project(':apps:i2ptunnel')
+    implementation project(':apps:jetty')
+    implementation project(':apps:i2psnark')
+    implementation project(':apps:systray')
+    implementation project(':apps:BOB')
+    implementation project(':apps:sam')
+    implementation project(':apps:routerconsole')
+    implementation project(':apps:desktopgui')
+    implementation project(':apps:jrobin')
+    implementation project(':apps:addressbook')
+    implementation project(':apps:susidns')
+    implementation project(':apps:susimail')
+    implementation project(':apps:i2pcontrol')
+    implementation project(':apps:imagegen')
+    implementation project(':core')
+    implementation project(':router')
+    implementation project(path : ':installer', configuration: 'jbigi')
+    implementation fileTree("../apps/susidns/src/lib")
+}
+
+import java.nio.file.*
+import java.util.zip.*
+
+def projects = [':apps:routerconsole',
+                ':apps:addressbook',
+                ':apps:i2psnark',
+                ':apps:imagegen',
+                ':apps:i2pcontrol',
+                ':apps:susidns',
+                ':apps:susimail',
+                ':apps:i2ptunnel'].collect {project(it)}
+
+def rootDir = project.getRootProject().getRootDir()
+def buildDir = new File("$buildDir")
+def webappDir = new File(buildDir, 'webapps')
+def geoipDir = new File(buildDir, "geoip")
+
+task copyWars {
+    doLast {
+        webappDir.mkdirs()
+        projects.each { p ->
+            p.configurations.archives.getArtifacts().getFiles().
+                filter({it.getName().endsWith('war')}).each { file ->
+                println "copying war $file exists ${file.exists()}"
+                File target = new File(webappDir, file.getName())
+                Files.copy(file.toPath(), target.toPath())
+            }
+        } 
+    }
+}
+
+projects.each { p ->
+    copyWars.dependsOn p.tasks['assemble']    
+}
+
+task copyGeoip() {
+    doLast {
+        geoipDir.mkdirs()
+        File target = new File(geoipDir, "GeoLite2-Country.mmdb")
+        File source = new File("$rootDir/installer/resources/GeoLite2-Country.mmdb.gz")
+        InputStream is = new GZIPInputStream(new FileInputStream(source))
+        java.nio.file.Files.copy(is, target.toPath())
+    }
+}
+
+jar.dependsOn copyWars,copyGeoip
+
+distributions {
+    main {
+        contents {
+            from('../installer/resources/') {
+                exclude '*.gz*'
+            }
+            from(webappDir.getAbsolutePath()) {
+                into 'webapps'
+            }
+            from(geoipDir.getAbsolutePath()) {
+                into 'geoip'
+            }
+        }
+    }
+}

settings.gradle

@@ -17,3 +17,4 @@ include 'apps:imagegen'
 include 'core'
 include 'installer'
 include 'router'
+include 'reproducible'