Allow loading migrations from a jar file

[fkoehler]

We are using Pillar inside a Play2 application where it is deployed as a jar. One can not read migrations from a directory inside a jar. We worked around this with something like the following. I am not creating a pull request as I am not sure on how to integrate this and if it's desired behaviour nor do I know how to correctly test this. sorry.

  private val registry = Registry(loadMigrationsFromJarOrFilesystem())
  private val migrator = Migrator(registry, new LoggerReporter)

  private def loadMigrationsFromJarOrFilesystem() = {
    val migrationsDir = "migrations/"
    val migrationNames = JarUtils.getResourceListing(getClass, migrationsDir).toList.filter(_.nonEmpty)
    val parser = Parser()

    migrationNames.map(name => getClass.getClassLoader.getResourceAsStream(migrationsDir + name)).map {
      stream =>
        try {
          parser.parse(stream)
        } finally {
          stream.close()
        }
    }.toList
  }

where JarUtils.getResourceListing is taken from the top answer here: http://stackoverflow.com/questions/6247144/how-to-load-a-folder-from-a-jar

Hope that helps

[magro]

@pvenable WDYT?

[pvenable]

I don't have this use case, but it seems reasonable to support it. :)

[PeterLappo]

Hi Have implemented some code that loads from a class path and directories as I have a need for this. ClassPathUtil.java and CPRegistry.scala. You might not like the logger or the package names. Sorry no unit tests but works fine in my project.

package util;

import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.ArrayList;
import java.util.Enumeration;
import java.util.List;
import java.util.jar.JarEntry;
import java.util.jar.JarFile;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * ClassPathUtil
 *  
 * @author Peter Lappo
 *
 */

public class ClassPathUtil {

  private static Logger log = LoggerFactory.getLogger(ClassPathUtil.class);

  /**
   * List directory contents from the classpath. Not recursive. This is basically a brute-force implementation. Works
   * for regular files and also JARs.
   * 
   * For example:
   * ClassPathUtil.getResourceListing(this.getClass(), "cql/migrations/");
   * 
   * @param clazz
   *          Any java class that lives in the same place as the resources you want.
   * @param path
   *          Must end with "/" otherwise path is not formed properly. 
   *          Should not start with "/" as we search from the root directory anyway.
   * @return List of URLs for each resource found in a resource directory or jar.
   * 
   * @throws URISyntaxException
   * @throws IOException
   * 
   * @author Peter Lappo
   */
  public static List<URL> getResourceListing(Class<?> clazz, String path) throws URISyntaxException, IOException {
    Enumeration<URL> dirURLs = clazz.getClassLoader().getResources(path);
    List<URL> result = new ArrayList<URL>();

    while (dirURLs.hasMoreElements()) {
      URL url = dirURLs.nextElement();
      log.info("Checking url: " + url);

      if (url.getProtocol().equals("file")) {
        File file = new File(url.getFile());
        if (file.isDirectory()) {
          // enumerate files in directory
          for (String entrystr :file.list()) {
            result.add(new URL(url.toExternalForm() + entrystr));
          }
        } else {
          result.add(url);
        }
      }

      if (url.getProtocol().equals("jar")) {
        if (url.toExternalForm().contains("sources")) {
          log.warn("Ignoring sources url: " + url);
          continue; // ignore sources
        }
        /* A JAR path */
        String jarPath = url.getPath().substring(5, url.getPath().indexOf("!")); // strip out only the JAR file
        JarFile jar = new JarFile(URLDecoder.decode(jarPath, "UTF-8"));
        Enumeration<JarEntry> entries = jar.entries(); // gives ALL entries in jar
        while (entries.hasMoreElements()) {
          JarEntry entry = entries.nextElement();
          String name = entry.getName();
          if (name.startsWith(path)) { // filter according to the path
            if (entry.isDirectory()) {
              log.warn("Ignoring subdirectory url: " + entry);
              continue; // ignore subdirectories
            }
            String entrystr = name.substring(path.length());
            result.add(new URL(url.toExternalForm() + entrystr));
          }
        }
        jar.close();
      }
    }

    return result;
  }

}

package util

import java.io.BufferedInputStream
import java.io.File
import java.net.URL
import com.chrisomeara.pillar.Migration
import com.chrisomeara.pillar.Parser
import com.chrisomeara.pillar.Registry
import com.chrisomeara.pillar.Reporter
import com.chrisomeara.pillar.ReportingMigration
import java.io.InputStream

/**
 * Pillar Registry factory the loads migration definitions from the classpath.
 * This will load definitions from files and jar resources.
 * 
 * @author Peter Lappo
 */

object CPRegistry {
  def apply(migrations: Seq[Migration]): Registry = {
    new Registry(migrations)
  }

  def fromClassPath(clazz: Class[_], path: String, reporter: Reporter): Registry = {
    new Registry(parseMigrationsInClassPath(path).map(new ReportingMigration(reporter, _)))
  }

  def fromClassPath(clazz: Class[_], path: String): Registry = {
    new Registry(parseMigrationsInClassPath(path))
  }

  private def parseMigrationsInClassPath(path: String): Seq[Migration] = {
    import scala.collection.JavaConversions.asScalaBuffer
    val paths = ClassPathUtil.getResourceListing(this.getClass(), path)
    val parser = Parser()
    paths.map {
      path =>
        val stream:InputStream = new BufferedInputStream(path.openStream())
        try {
          parser.parse(stream)
        } finally {
          stream.close()
        }
    }.toList
  }

}

[magro]

@PeterLappo Why does fromClassPath require a clazz? It's not used AFAICS.

@comeara Would you merge a pull request with this?

[PeterLappo]

Enumeration dirURLs = clazz.getClassLoader().getResources(path);

The JVM can have more than one classloader to avoid conflicts in class versions (think webserver frameworks) and is used to manage security so you need to load the resources for your context. The chances are it would work fine without one.

Peter

www.smr.co.uk +44 7767 784452

On 2 Nov 2014, at 18:59, Martin Grotzke notifications@github.com wrote:

@PeterLappo Why does fromClassPath require a clazz? It's not used AFAICS.

@comeara Would you merge a pull request with this?

— Reply to this email directly or view it on GitHub.

[magro]

Sure, but shouldn't fromClassPath pass the clazz to parseMigrationsInClassPath then? Alternatively it could accept a classloader.

Cheers, Martin Am 02.11.2014 18:54 schrieb "Peter" notifications@github.com:

Hi Have implemented some code that loads from a class path and directories as I have a need for this. ClassPathUtil.java and CPRegistry.scala. You might not like the logger or the package names. Sorry no unit tests but works fine in my project.

package util; import java.io.File;import java.io.IOException;import java.net.URISyntaxException;import java.net.URL;import java.net.URLDecoder;import java.util.ArrayList;import java.util.Enumeration;import java.util.List;import java.util.jar.JarEntry;import java.util.jar.JarFile; import org.slf4j.Logger;import org.slf4j.LoggerFactory; /* * ClassPathUtil * * @author Peter Lappo * / public class ClassPathUtil {

private static Logger log = LoggerFactory.getLogger(ClassPathUtil.class);

/* * List directory contents from the classpath. Not recursive. This is basically a brute-force implementation. Works * for regular files and also JARs. * * For example: * ClassPathUtil.getResourceListing(this.getClass(), "cql/migrations/"); * * @param clazz * Any java class that lives in the same place as the resources you want. * @param path * Must end with "/" otherwise path is not formed properly. * Should not start with "/" as we search from the root directory anyway. * @return List of URLs for each resource found in a resource directory or jar. * * @throws URISyntaxException * @throws IOException * * @author Peter Lappo / public static List getResourceListing(Class<?> clazz, String path) throws URISyntaxException, IOException { Enumeration dirURLs = clazz.getClassLoader().getResources(path); List result = new ArrayList();

while (dirURLs.hasMoreElements()) {
  URL url = dirURLs.nextElement();
  log.info("Checking url: " + url);

  if (url.getProtocol().equals("file")) {
    File file = new File(url.getFile());
    if (file.isDirectory()) {
      // enumerate files in directory
      for (String entrystr :file.list()) {
        result.add(new URL(url.toExternalForm() + entrystr));
      }
    } else {
      result.add(url);
    }
  }

  if (url.getProtocol().equals("jar")) {
    if (url.toExternalForm().contains("sources")) {
      log.warn("Ignoring sources url: " + url);
      continue; // ignore sources
    }
    /* A JAR path */
    String jarPath = url.getPath().substring(5, url.getPath().indexOf("!")); // strip out only the JAR file
    JarFile jar = new JarFile(URLDecoder.decode(jarPath, "UTF-8"));
    Enumeration<JarEntry> entries = jar.entries(); // gives ALL entries in jar
    while (entries.hasMoreElements()) {
      JarEntry entry = entries.nextElement();
      String name = entry.getName();
      if (name.startsWith(path)) { // filter according to the path
        if (entry.isDirectory()) {
          log.warn("Ignoring subdirectory url: " + entry);
          continue; // ignore subdirectories
        }
        String entrystr = name.substring(path.length());
        result.add(new URL(url.toExternalForm() + entrystr));
      }
    }
    jar.close();
  }
}

return result;

} }

package util import java.io.BufferedInputStreamimport java.io.Fileimport java.net.URLimport com.chrisomeara.pillar.Migrationimport com.chrisomeara.pillar.Parserimport com.chrisomeara.pillar.Registryimport com.chrisomeara.pillar.Reporterimport com.chrisomeara.pillar.ReportingMigrationimport java.io.InputStream /* * Pillar Registry factory the loads migration definitions from the classpath. * This will load definitions from files and jar resources. * * @author Peter Lappo / object CPRegistry { def apply(migrations: Seq[Migration]): Registry = { new Registry(migrations) }

def fromClassPath(clazz: Class[], path: String, reporter: Reporter): Registry = { new Registry(parseMigrationsInClassPath(path).map(new ReportingMigration(reporter, ))) }

def fromClassPath(clazz: Class[_], path: String): Registry = { new Registry(parseMigrationsInClassPath(path)) }

private def parseMigrationsInClassPath(path: String): Seq[Migration] = { import scala.collection.JavaConversions.asScalaBuffer val paths = ClassPathUtil.getResourceListing(this.getClass(), path) val parser = Parser() paths.map { path => val stream:InputStream = new BufferedInputStream(path.openStream()) try { parser.parse(stream) } finally { stream.close() } }.toList } }

— Reply to this email directly or view it on GitHub https://github.com/comeara/pillar/issues/9#issuecomment-61416190.

[ches]

I agree it would be nice to be able to use migrations from resources on the classpath.

This alternative that functions as a library allows you to deploy your usual artifact instead of needing to somehow also deploy source (or a standalone repo of conf/) + the pillar executable into a server environment where it can reach the production Cassandra cluster. Support for this would pave the way for more easily using Pillar as a library for the same sort of use case when required.