How to custom a classloader - classloader

I'm trying to implementing this function with a customer classloader: I have some class files in a alternatives.jar file, they provide different implementation than normal implementation. i.e., each class in this jar, has another version which in other jar file -- also get loaded in the classpath.
I know it's better to use instrument API to achieve same purpose. But now my concern is I need to understand why I'm failing.
So this is my method:
1. define a AlternativeClassLoader.java, in this file, I override findClass method. So if the class name can be found from alternatives.jar, then use the version from alternatives.jar.
2. in constructor, I have called super(null) so all these class loading work will be performed by my classloader, rather that system's.
3. This (seems to be true) also requires me to load other classes (if they're not system one). So I have to parse classpath, find all classes which indicated by the classpath.
My problem is, I can load my alternative class, everything seems to be fine...However, I'm using slf4j which yells the following error:
Failed to auto configure default logger context
Reported exception:
ch.qos.logback.core.joran.spi.JoranException: Parser configuration error occurred
Failed to instantiate [ch.qos.logback.classic.LoggerContext]
Reported exception:
java.lang.ExceptionInInitializerError
at java.util.ResourceBundle.getLoader(ResourceBundle.java:431)
at java.util.ResourceBundle.getBundle(ResourceBundle.java:841)
I doubt this is caused by my bad classloader implementation. Would somebody help me out? Many thanks!
This is my classloader:
public class AlternativeClassLoader extends ClassLoader {
private static final String ALTERNATIVE_JAR_PROPERTY = "alternativejar";
private static final Logger logger = LoggerFactory
.getLogger(AlternativeClassLoader.class);
private Map<String, Class<?>> clzCache = new HashMap<String, Class<?>>();
private Map<String, String> others = new HashMap<String, String>();
private Set<String> alternativesRegistry;
private JarFile altjar;
public AlternativeClassLoader(ClassLoader parent) {
/*
* pass null so I can incept all class loading except system's. By doing
* this you'll need to override findClass
*/
super(null);
registerAlternatives();
registerOthers();
}
/**
* This method will parse classpath and get all non-system class name, and
* build classname - jar_file_path/file_system_path mappings
*/
private void registerOthers() {
String[] paths = System.getProperty("java.class.path").split(":");
URL[] urls = new URL[paths.length];
for (String path : paths) {
if (path.endsWith("*.jar")) {
registerClass(path, others);
} else {
File f = new File(path);
if (!f.isDirectory())
continue;
File[] classFiles = f.listFiles(new FileFilter() {
#Override
public boolean accept(File arg0) {
if (arg0.getName().endsWith(".class")) {
return true;
} else {
return false;
}
}
});
for (File file : classFiles) {
String fileName = file.getName();
String className = fileName.substring(0,
fileName.lastIndexOf("."));
others.put(className, file.getPath());
}
}
}
showRegistry(
"Me will also be responsible for loading the following classes:",
others);
}
private void registerClass(String path, Map<String, String> registry) {
try {
JarInputStream jis = new JarInputStream(new FileInputStream(path));
for (JarEntry entry = jis.getNextJarEntry(); entry != null; entry = jis
.getNextJarEntry()) {
if (entry.getName().endsWith(".class") && !entry.isDirectory()) {
StringBuilder className = new StringBuilder();
for (String part : entry.getName().split("/")) {
if (className.length() != 0)
className.append(".");
className.append(part);
if (part.endsWith(".class"))
className.setLength(className.length()
- ".class".length());
}
registry.put(className.toString(), path);
}
}
} catch (Exception e) {
e.printStackTrace(System.out);
logger.error(
"Failed when read/parse jar {}. Your class file may not been replaced by alternative implementation",
path, e);
}
}
/**
* Try to find alternative class implementation from jar file specified by
* ALTERNATIVE_JAR_PROPERTY. If it's not specified, then use same jar file
* where this classloader is loaded.
*/
private void registerAlternatives() {
String jarFilePath = System.getProperty(ALTERNATIVE_JAR_PROPERTY);
if (jarFilePath == null || jarFilePath.isEmpty()) {
URL url = getClass().getProtectionDomain().getCodeSource()
.getLocation();
System.out.println(url + ":" + url.toString());
jarFilePath = url.getPath();
}
try {
altjar = new JarFile(jarFilePath);
} catch (IOException e) {
logger.error("cannot read jar {}", jarFilePath);
return;
}
Map<String, String> registry = new HashMap<String, String>();
registerClass(jarFilePath, registry);
alternativesRegistry = registry.keySet();
showRegistry("===Found the following alternative class:===", registry);
}
private void showRegistry(String string, Map<String, String> registry) {
System.out.println(string);
for (String clzName : registry.keySet()) {
System.out.printf("Class:%30s ->%s\n", clzName,
registry.get(clzName));
}
}
private Class<?> myLoadClass(String name) throws IOException,
ClassFormatError {
logger.debug("myload class {}", name);
System.out.printf("myload class %s\n", name);
if (alternativesRegistry.contains(name) && altjar != null) {
JarEntry entry = altjar.getJarEntry(name + ".class");
InputStream is = altjar.getInputStream(entry);
return readClassData(name, is);
}
String path = others.get(name);
if (path == null || path.isEmpty()) {
return null;
}
if (path.endsWith(".jar")) {
JarFile jar = new JarFile(path);
JarEntry entry = jar.getJarEntry(name + ".class");
InputStream is = jar.getInputStream(entry);
return readClassData(name, is);
} else {// it's a folder, need to read clz from .class file
System.out.printf("file path for %s is %s\n", name, path);
InputStream is = new FileInputStream(new File(path));
return readClassData(name, is);
}
}
private Class<?> readClassData(String name, InputStream is)
throws IOException, ClassFormatError {
byte[] buffer = new byte[4096];
ByteArrayOutputStream out = new ByteArrayOutputStream(buffer.length);
int len = is.read(buffer);
while (len > 0) {
out.write(buffer, 0, len);
len = is.read(buffer);
}
Class<?> clz = defineClass(name, out.toByteArray(), 0, out.size());
if (clz != null) {
System.out.printf("loaded %s by me\n", name);
clzCache.put(name, clz);
}
return clz;
}
protected Class<?> findCachedClass(String name)
throws ClassNotFoundException {
Class<?> clz = clzCache.get(name);
if (clz == null) {
clz = findLoadedClass(name);
}
return clz;
}
#Override
protected Class<?> findClass(String name) throws ClassNotFoundException {
System.out.println("findClass: " + name);
Class<?> cls = findCachedClass(name);
if (cls == null) {
try {
cls = myLoadClass(name);
} catch (ClassFormatError | IOException e) {
logger.error("failed to load class {}", name, e);
System.out.printf("failed to load class %s\n", name);
e.printStackTrace();
}
}
return cls;
}
}
I have tried to override findResource(), but it's never called.
This is how I put my classloader into use:
java -Djava.system.class.loader=AlternativeClassLoader -classpath=.:./alternatives.jar:./slf4j-xxx.jar Test

OK, I solved the problem. The tricky is:
Never use any package other than java.*. Otherwise, it will cause recursively loading ...IllegalState error.
In your classloader constructor, load all the alternative class and cache them.
In your constructor, call super(parent) other than super(null), then you don't need to do all the class loading stuff, the parent classloader can do it for you.
in override findClass(), if the class can be found from cache (means they have alternative implementation), then return it, otherwise let super.findClass do the rest for you.
so the following is the source code:
public class AlternativeClassLoader extends ClassLoader {
private static final String ALTERNATIVE_JAR_PROPERTY = "alternativejar";
private Map<String, Class<?>> clzCache = new HashMap<String, Class<?>>();
public AlternativeClassLoader(ClassLoader parent) {
super(parent);
loadAlternativeClasses();
}
private void loadAlternativeClasses() {
String jarFilePath = System.getProperty(ALTERNATIVE_JAR_PROPERTY);
if (jarFilePath == null || jarFilePath.isEmpty()){
URL url = getClass().getProtectionDomain().getCodeSource().getLocation();
System.out.println(url + ":" + url.toString());
jarFilePath = url.getPath();
}
JarInputStream jis;
try {
jis = new JarInputStream(new FileInputStream(jarFilePath));
JarEntry entry;
while ((entry = jis.getNextJarEntry()) != null){
String className = entry.getName();
className = className.substring(0, className.length() - ".class".length());
System.out.printf("loading class from %s: %s\n", jarFilePath, className);
readClassData(className, jis);
}
} catch (IOException e) {
e.printStackTrace();
}
} private Class<?> readClassData(String name, InputStream is) throws IOException,
ClassFormatError {
byte[] buffer = new byte[4096];
ByteArrayOutputStream out = new ByteArrayOutputStream(buffer.length);
int len = is.read(buffer);
while (len > 0) {
out.write(buffer, 0, len);
len = is.read(buffer);
}
Class<?> clz = defineClass(name, out.toByteArray(), 0, out.size());
if (clz != null) {
System.out.printf("loaded %s by myself\n", name);
clzCache.put(name, clz);
}
return clz;
}
protected Class<?> findClass(String name) throws ClassNotFoundException {
System.out.println("findClass: " + name);
Class<?> cls = clzCache.get(name);
if (cls == null)
cls = super.findClass(name);
return cls;
}
}

Related

Unloading Java classes using CustomClassLoader

I have been through many posts and questions but none of them had a absolute Java program to implement this.
Requirement : For some reasons my application loads the Common-codec 1.3.jar,
and later on ,within same jvm the process needs to use Common-code 1.10.jar which is of different version.
But since the previous classes are loaded and they have same package name ,reloading them with a Java program is not replacing the existing classes.
Here is the code (sample) i used to reload(replace) the existing keys,but found no luck as expected. Please let me know hoe this could be done by a Java example.
String pathToJar=/root/Desktop/commons-codec-1.10.jar";
JarFile jarFile = null;
try {
jarFile = new JarFile(pathToJar);
} catch (IOException e) {
e.printStackTrace();
}
Enumeration<JarEntry> e = jarFile.entries();
URL[] urls = new URL[0];
try {
urls = new URL[]{ new URL("jar:file:" + pathToJar+"!/") };
} catch (MalformedURLException e1) {
e1.printStackTrace();
}
URLClassLoader cl = URLClassLoader.newInstance(urls);
while (e.hasMoreElements()) {
JarEntry je = e.nextElement();
if(je.isDirectory() || !je.getName().endsWith(".class")){
continue;
}
// -6 because of .class
String className = je.getName().substring(0,je.getName().length()-6);
className = className.replace(File.separatorChar, '.');
String check="org.apache.commons.codec.binary.Base32";
try {
Class c = cl.loadClass(className); // Excepting it to replace old files,but thats not happening
} catch (ClassNotFoundException e1) {
e1.printStackTrace();
}
I was advised to write Custom class loader and unloading is possible via it.
Could some show some relevant code or process of doing it.
Class "unloading" may only occur when both the Class and the ClassLoader that defined it are eligible for garbage collection. Furthermore, any given loader can only define a class of a specific name once. Therefore (save for exotic workarounds such as instrumentation or JDK implementation internals), there is no such thing as a "reloading-capable class loader" -- at least not in the true sense of the term; in order to obtain n distinct "versions" of a class, C, defineClass(C) must be invoked on n distinct ClassLoader objects.
Let's examine the following trivial use case: We have an app.jar, (unsurprisingly) containing a class declaring a main method; and two copies of a lib.jar, each having a single class, LibApiImpl, encapsulating a single method that just prints a version-dependent string to stdout. Our goal is to merely reference two copies of the "library" class from our main and see two different strings being output. The remainder of this post demonstrates just two of all the possible approaches to satisfy this requirement.
Approach #1 - Manually instantiating ClassLoaders
The straightforward solution is to simply create a new URLClassLoader every time a different LibApiImpl needs to be loaded. No custom class loader implementation is required, and the default parent-first delegation model serves both the application and the library well. A couple of things to note:
The library JARs must not sit on the classpath, in order to not be discoverable by the default application class loader. Otherwise the application class loader will "favor" one over the other, and, due to the default delegation model, standard URLClassLoader children of the application loader will not be able to override their parent's opinion on the matter.
To access the library class, the 3-argument Class::forName must be employed, specifying the manually-instantiated class loader. The single-arg version will delegate to the application class loader, which (as per the former point) is of course unaware of the existence of the corresponding .class file.
If the reflectively-obtained class is to be cast to an interface, as will be the case herein, the interface must, as opposed to the implementation, reside on the classpath.
Demo code
package com.example.app;
import java.net.URL;
import java.net.URLClassLoader;
import com.example.lib.api.LibApi;
public class App {
public static void main(String... args) throws Exception {
try (URLClassLoader loader = new URLClassLoader(new URL[] { new URL("file:/path/to/lib1.jar") })) {
((LibApi) Class.forName("com.example.lib.impl.LibApiImpl", true, loader).newInstance()).printVersionInfo();
}
try (URLClassLoader loader = new URLClassLoader(new URL[] { new URL("file:/path/to/lib2.jar") })) {
((LibApi) Class.forName("com.example.lib.impl.LibApiImpl", true, loader).newInstance()).printVersionInfo();
}
}
}
package com.example.lib.api;
public interface LibApi {
void printVersionInfo();
}
package com.example.lib.impl;
import com.example.lib.api.LibApi;
public class LibApiImpl implements LibApi {
#Override
public void printVersionInfo() {
System.out.println("\n** lib " + getClass() + " / loaded by " + getClass().getClassLoader() + " **\n");
}
}
Packaging
Correct the paths in com.example.app.App; then produce the following 4 JARs:
An app.jar containing the com.example.app package.
A lib-api.jar containing the com.example.lib.api package.
Two "versions" (just export twice), lib1.jar and lib2.jar, each containing the com.example.lib.impl package.
Testing
Run as follows:
java -cp '/path/to/app.jar:/path/to/lib-api.jar' com.example.app.App
Sample output:
** lib class com.example.lib.impl.LibApiImpl / loaded by java.net.URLClassLoader#55f96302 **
** lib class com.example.lib.impl.LibApiImpl / loaded by java.net.URLClassLoader#135fbaa4 **
Approach #2 - Creating a "Master-Slave" loader
Sometimes the "parent-first" model is insufficient. As a contrived example, suppose that we wanted to obtain the "copy" of LibApiImpl that was last loaded by some child of the application class loader, without having to care about which child it actually was that defined the copy. In other words, we want the call Class.forName("com.example.lib.impl.LibApiImpl") to return the "most fresh" LibApiImpl version. But, unless we override the application class loader, or, more generally, the class loader of the App class, with a custom implementation, that call will always fail, because, under the default delegation model, delegation flows unidirectionally from lower-level loaders to their ancestors and not vice versa.
The application class loader implementation, given below, behaves as follows (refer to the Javadoc overview of the Loaders class for a concrete explanation): There is a "master" loader, serving as the application class loader, that may have a child, referred to as its "slave". The master is responsible for loading non-reloadable application classes from the classpath (app.jar and lib-api.jar, in this case), while the slave gets to load reloadable, non-classpath-residing ones (lib1.jar and lib2.jar). The communication between the two is bidirectional, and the loader that will ultimately define any given class, "fixed" or "reloadable", is always respectively the master and the slave, regardless the "initiating" loader, i.e., the loader on which the application called loadClass, or passed to Class::forName. Of course this is nothing but a toy implementation meant to (hopefully) illustrate how a different delegation scheme might look like, and is probably flawed in ways I have not even begun to imagine. A real-world implementation would, e.g., have to provide proper concurrency; provide compliant implementations of getResource et al.; address concerns of code accessibility, verification, and possibly code base privilege assignment; and allow extensibility by, and configuration of, multiple slaves, or even attachment of other children, potentially of arbitrary implementation, to the master, while preserving well-defined delegation semantics. The point of course being that it is generally too much effort to write a proper implementation, when there is OSGi and the like doing just that (among many others).
Demo code
package com.example.app;
import java.net.URL;
import com.example.app.Loaders.MasterLoader;
import com.example.lib.api.LibApi;
public class App2 {
public static void main(String... args) throws Exception {
MasterLoader loader = (MasterLoader) ClassLoader.getSystemClassLoader();
loader.setDebug(true);
loader.refresh(new URL[] { new URL("file:/path/to/lib1.jar") });
newLibApi().printVersionInfo();
loader.refresh(new URL[] { new URL("file:/path/to/lib2.jar") });
newLibApi().printVersionInfo();
}
static LibApi newLibApi() throws Exception {
return (LibApi) Class.forName("com.example.lib.impl.LibApiImpl").newInstance();
}
}
package com.example.app;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
import java.nio.file.Paths;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.regex.Pattern;
/**
* Contains a "Master (parent)/Slave (child)" <code>ClassLoader</code> "duo".<br/>
* <br/>
* The class loading "protocol" adhered by the respective {#code loadClass(String, boolean)}
* implementations of the two is as follows:
* <ol>
* <li>If the name argument matches the name of the loader's class itself, return that class.
* Otherwise proceed.</li>
* <li>If a call to {#code findLoadedClass(name)} yields a non-null class, return that class.
* Otherwise proceed.</li>
* <li>Let <em>C</em> be either this loader's parent, if this is a "slave", or its child, if this is
* a "master". If <em>C</em> is non-null, have it execute steps (1), (2) itself. If a class gets
* produced, return that class. Otherwise (i.e., if there either is no result, or <em>C</em> is
* null) proceed.</li>
* <li>If the name argument refers to a potential bootstrap classpath class name, call
* {#code loadClass(name)} on the default system classloader (the "master's" parent). If the call
* succeeds, return that class. Otherwise proceed.</li>
* <li>If the name argument refers to a .class file under this loader's search path, read it, define
* and link a new class off of its contents, and return that "freshly-fefined" class. Otherwise
* proceed.</li>
* <li>Once again, let <em>C</em> be the loader specified in step (3). If non-null, have it execute
* step (5) on itself. If a class gets produced, return that class. Otherwise fail.</li>
* </ol>
*/
public class Loaders {
private static class SlaveLoader extends URLClassLoader {
static final Pattern BOOT_CLASS_PATH_RES_NAMES = Pattern.compile("((com\\.)?sun|java(x)?)\\..*");
static final URL[] EMPTY_SEARCH_PATH = new URL[0];
static final URL[] createSearchPath(String pathNames) {
if (pathNames != null) {
List<URL> searchPath = new ArrayList<>();
for (String pathName : pathNames.split(File.pathSeparator)) {
try {
searchPath.add(Paths.get(pathName).toUri().toURL());
}
catch (MalformedURLException e) {
e.printStackTrace();
}
}
return searchPath.toArray(new URL[0]);
}
return EMPTY_SEARCH_PATH;
}
static final byte[] readClassData(URL classResource) throws IOException {
try (InputStream in = classResource.openStream(); ByteArrayOutputStream out = new ByteArrayOutputStream()) {
while (in.available() > 0) {
out.write(in.read());
}
return out.toByteArray();
}
}
final String loadClassOutcomeMsgFmt = "loadClass return '{'\n\tloader = {0}\n\ttarget = {1}\n\tresult : {2}\n'}'";
volatile boolean debug;
SlaveLoader(URL[] searchPath, ClassLoader parent) {
super(searchPath, parent);
}
#Override
protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
validateName(name);
Class<?> ret = loadFromCache(name);
if (ret != null) {
return ret;
}
MasterLoader parent = (MasterLoader) getParent();
if ((ret = parent.loadFromCache(name)) != null) {
log(loadClassOutcomeMsgFmt, this, name, "3 - early return - pre-loaded/cached - via " + parent);
return ret;
}
if ((ret = loadFromBootClasspath(name)) != null) {
return ret;
}
if ((ret = loadFromSearchPath(name, resolve)) != null) {
return ret;
}
if ((ret = parent.loadFromSearchPath(name, resolve)) != null) {
log(loadClassOutcomeMsgFmt, this, name,
"6 - common/non-reloadable classpath delegation - via " + parent);
return ret;
}
if ((ret = parent.loadFromSearchPath(name, resolve)) != null) {
return ret;
}
throw createCnfe(name, null);
}
void validateName(String name) throws ClassNotFoundException {
log("loadClass entry '{'\n\tloader = {0}\n\ttarget = {1}\n'}'", this, name);
if ((name == null) || name.trim().isEmpty()) {
throw createCnfe(name, null);
}
}
Class<?> loadFromCache(String name) {
Class<?> ret = getClass();
if (ret.getName().equals(name)) {
log(loadClassOutcomeMsgFmt, this, name, "1 - early return - own class");
return ret;
}
if ((ret = findLoadedClass(name)) != null) {
log(loadClassOutcomeMsgFmt, this, name, "2 - early return - pre-loaded/cached");
return ret;
}
return null;
}
Class<?> loadFromBootClasspath(String name) {
if (BOOT_CLASS_PATH_RES_NAMES.matcher(name).matches()) {
ClassLoader defSysCl = ClassLoader.getSystemClassLoader().getParent();
try {
Class<?> ret = ClassLoader.getSystemClassLoader().getParent().loadClass(name);
log(loadClassOutcomeMsgFmt, this, name, "4 - bootstrap classpath delegation - via " + defSysCl);
return ret;
}
catch (ClassNotFoundException cnfe) {
}
}
return null;
}
Class<?> loadFromSearchPath(String name, boolean resolve) throws ClassNotFoundException {
Class<?> ret = null;
URL res = findResource(name.replace(".", "/") + ".class");
if (res != null) {
byte[] b;
try {
b = readClassData(res);
}
catch (IOException ioe) {
throw createCnfe(name, ioe);
}
ret = defineClass(name, b, 0, b.length);
if (resolve) {
resolveClass(ret);
}
log(loadClassOutcomeMsgFmt, this, name, "5 - freshly-defined from local search path");
return ret;
}
return null;
}
ClassNotFoundException createCnfe(String name, Throwable cause) throws ClassNotFoundException {
return new ClassNotFoundException(MessageFormat.format("Class loading : {0} : {1} : FAILED", this,
(name == null) ? "null" : name, cause));
}
void log(String msg, Object... args) {
if (debug) {
System.out.println(MessageFormat.format("\n" + msg + "\n", args));
}
}
public void setDebug(boolean debug) {
this.debug = debug;
}
#Override
protected void finalize() throws Throwable {
try {
close();
}
finally {
super.finalize();
}
log("ClassLoader finalization : {0}", this);
}
}
public static class MasterLoader extends SlaveLoader {
static final URL[] DEFAULT_CLASS_PATH = createSearchPath(System.getProperty("java.class.path"));
private URL[] reloadableSearchPath = EMPTY_SEARCH_PATH;
private volatile SlaveLoader slave;
public MasterLoader(ClassLoader parent) {
super(DEFAULT_CLASS_PATH, parent);
}
public synchronized void refresh(URL[] reloadableSearchPath) {
int len;
if ((reloadableSearchPath != null) && ((len = reloadableSearchPath.length) > 0)) {
List<URL> path = new ArrayList<>(len + 1);
for (int i = 0; i < len; i++) {
URL entry = reloadableSearchPath[i];
if (entry != null) {
path.add(entry);
}
}
this.reloadableSearchPath = (!path.isEmpty()) ? path.toArray(EMPTY_SEARCH_PATH) : EMPTY_SEARCH_PATH;
}
else {
this.reloadableSearchPath = EMPTY_SEARCH_PATH;
}
if (slave != null) {
try {
slave.close();
}
catch (IOException ioe) {
}
slave = null;
/*
* At least two calls to System::gc appear to be required in order for Class::forName to cease
* returning cached classes previously defined by slave and for which the master served as an
* intermediary, i.e., an "initiating loader".
*
* See also http://blog.hargrave.io/2007/09/classforname-caches-defined-class-in.html
*/
for (int i = 0; i < 2; i++) {
System.gc();
try {
Thread.sleep(100);
}
catch (InterruptedException ie) {
}
}
}
if (this.reloadableSearchPath != EMPTY_SEARCH_PATH) {
log("Class loader search path refresh : {0}\n\tSearch path = {1}", this,
Arrays.toString(this.reloadableSearchPath));
slave = new SlaveLoader(this.reloadableSearchPath, this);
slave.setDebug(debug);
}
}
#Override
protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
validateName(name);
Class<?> ret = loadFromCache(name);
if (ret != null) {
return ret;
}
if ((slave != null) && ((ret = slave.loadFromCache(name)) != null)) {
log(loadClassOutcomeMsgFmt, this, name, "3 - early return - pre-loaded/cached - via " + slave);
return ret;
}
if ((ret = loadFromBootClasspath(name)) != null) {
return ret;
}
if ((ret = loadFromSearchPath(name, resolve)) != null) {
return ret;
}
if ((slave != null) && ((ret = slave.loadFromSearchPath(name, resolve)) != null)) {
log(loadClassOutcomeMsgFmt, this, name,
"6 - reloadable classpath delegation - via " + ret.getClassLoader());
return ret;
}
throw createCnfe(name, null);
}
}
}
Packaging
Once again correct the paths in com.example.app.App2; add App2 and com.example.app.Loaders to app.jar; and re-export. The other JARs should remain as in the former example.
Testing
Run as follows:
java -cp '/path/to/app.jar:/path/to/lib-api.jar' \
'-Djava.system.class.loader=com.example.app.Loaders$MasterLoader' \
com.example.app.App2
Sample output (loadClass debug omitted):
** lib class com.example.lib.impl.LibApiImpl / loaded by com.example.app.Loaders$SlaveLoader#7f31245a **
...
ClassLoader finalization : com.example.app.Loaders$SlaveLoader#7f31245a
...
** lib class com.example.lib.impl.LibApiImpl / loaded by com.example.app.Loaders$SlaveLoader#12a3a380 **

Spring Data Neo4j 4 : Failed to convert from type java.util.LinkedHashSet<?> to type org.springframework.data.domain.Page<?>

Having this Repository method
#Query("MATCH (i:`Interest`) WHERE not(i-[:PARENT]->()) return i")
public Page<Interest> findAllByParentIsNull(Pageable pageRequest);
It cause (it didn't respect the specification):
org.springframework.core.convert.ConversionFailedException: Failed to convert from type java.util.LinkedHashSet<?> to type org.springframework.data.domain.Page<?> for value '[com.nearofme.model.Interest#12a4479, com.nearofme.model.Interest#15bdfb3, com.nearofme.model.Interest#1af6067, com.nearofme.model.Interest#1c17d4d, com.nearofme.model.Interest#df65f4, com.nearofme.model.Interest#3b140d, com.nearofme.model.Interest#1e24566, com.nearofme.model.Interest#da49c9]'; nested exception is org.springframework.core.convert.ConverterNotFoundException: No converter found capable of converting from type java.util.LinkedHashSet<?> to type org.springframework.data.domain.Page<?>
at org.springframework.core.convert.support.ConversionUtils.invokeConverter(ConversionUtils.java:41)
at org.springframework.core.convert.support.GenericConversionService.convert(GenericConversionService.java:192)
at org.springframework.core.convert.support.GenericConversionService.convert(GenericConversionService.java:176)
at org.springframework.data.repository.core.support.QueryExecutionResultHandler.postProcessInvocationResult(QueryExecutionResultHandler.java:75)
at org.springframework.data.repository.core.support.RepositoryFactorySupport$QueryExecutorMethodInterceptor.invoke(RepositoryFactorySupport.java:439)
Debugging the code shows that a conversion is needed at the GraphRepositoryQuery level :
#Override
public final Object execute(Object[] parameters) {
Class<?> returnType = graphQueryMethod.getMethod().getReturnType();
Class<?> concreteType = graphQueryMethod.resolveConcreteReturnType();
Map<String, Object> params = resolveParams(parameters);
// could be converted here
return execute(returnType, concreteType, getQueryString(), params);
}
The current code convert the result at GraphRepositoryImpl with the private method updatePage that should be used in the GraphRepositoryQuery
#Override
public Page<T> findAll(Pageable pageable, int depth) {
Collection<T> data = session.loadAll(clazz, convert(pageable.getSort()), new Pagination(pageable.getPageNumber(), pageable.getPageSize()), depth);
return updatePage(pageable, new ArrayList<T>(data));
}
So my current temporary solution is to update GraphRepositoryQuery with :
#Override
public final Object execute(Object[] parameters) {
Class<?> returnType = graphQueryMethod.getMethod().getReturnType();
Class<?> concreteType = graphQueryMethod.resolveConcreteReturnType();
Map<String, Object> params = resolveParams(parameters);
Object result = execute(returnType, concreteType, getQueryString(), params);
if (params.size()>0){
Object param = params.values().toArray()[0];
if (param instanceof Pageable){
Pageable pageable = (Pageable) param;
result = updatePage(pageable, new ArrayList((Collection) result));
}
}
return result;
}
private Page updatePage(Pageable pageable, List results) {
int pageSize = pageable.getPageSize();
int pageOffset = pageable.getOffset();
int total = pageOffset + results.size() + (results.size() == pageSize ? pageSize : 0);
return new PageImpl(results, pageable, total);
}
No more need to convert inside the GraphRepositoryImpl but it still working

MapReduce Hadoop - mapside joining the 2 data sets using customInputFormat

I am learning hadoop mapreduce framework ,i am trying to join 2 data sets have first record(Text) in the line as the Key , i tried to search in stackoverflow previous posts but nothing worked out.Here i am trying to customize the InputFormat and trying to join with the ID which is first record in the each line of data set.
My input file1:
cdd8dde3-0349-4f0d-b97a-7ae84b687f9c,Esther,Garner,4071 Haven Lane,Okemos,MI
81a43486-07e1-4b92-b92b-03d0caa87b5f,Timothy,Duncan,753 Stadium Drive,Taunton,MA
File2 :
cdd8dde3-0349-4f0d-b97a-7ae84b687f9c,517-706-9565,EstherJGarner#teleworm.us,Waskepter38,noL2ieghie,MasterCard,5305687295670850
81a43486-07e1-4b92-b92b-03d0caa87b5f,508-307-3433,TimothyDDuncan#einrot.com,Conerse,Gif4Edeiba,MasterCard,5265896533330445
**Driver class:**
conf.setInputFormat(CompositeInputFormat.class);
String strJoinStmt = CompositeInputFormat.compose("inner",
KeyValueLongInputFormat.class, dirEmployeesData, dirSalaryData);
conf.set("mapred.join.expr", strJoinStmt);
conf.setNumReduceTasks(0);
dirOutput.getFileSystem(conf).delete(dirOutput);
TextOutputFormat.setOutputPath(conf, dirOutput);
conf.setOutputKeyClass(Text.class);
conf.setOutputValueClass(Text.class);
conf.setOutputFormat(TextOutputFormat.class);
**Custom RecordReader class:**
public class KeyValueLongLineRecordReader implements
RecordReader<Text, Text> {
private final LineRecordReader lineRecordReader;
private byte separator = (byte) ',';
private LongWritable dummyKey;
private Text innerValue;
public Class getKeyClass() {
return Text.class;
}
public Text createKey() {
return new Text("");
}
public Text createValue() {
return new Text();
}
public KeyValueLongLineRecordReader(Configuration job, FileSplit split)
throws IOException {
lineRecordReader = new LineRecordReader(job, split);
dummyKey = lineRecordReader.createKey();
innerValue = lineRecordReader.createValue();
String sepStr = job.get("key.value.separator.in.input.line", ",");
this.separator = (byte) sepStr.charAt(0);
}
public static int findSeparator(byte[] utf, int start, int length, byte sep) {
for (int i = start; i < (start + length); i++) {
if (utf[i] == sep) {
return i;
}
}
return -1;
}
/** Read key/value pair in a line. */
public synchronized boolean next(Text key, Text value)
throws IOException {
Text tKey = key;
Text tValue = value;
byte[] line = null;
int lineLen = -1;
if (!lineRecordReader.next(dummyKey, innerValue)) {
return false;
}
else
line = innerValue.getBytes();
lineLen = innerValue.getLength();
if (line == null)
return false;
int pos = findSeparator(line, 0, lineLen, this.separator);
if (pos == -1) {
tKey.set(new String(line, 0, lineLen));
tValue.set("");
} else {
int keyLen = pos;
byte[] keyBytes = new byte[keyLen];
System.arraycopy(line, 0, keyBytes, 0, keyLen);
int valLen = lineLen - keyLen - 1;
byte[] valBytes = new byte[valLen];
System.arraycopy(line, pos + 1, valBytes, 0, valLen);
tKey.set(new String(keyBytes));
tValue.set(valBytes);
}
return true;
}
}
**InputFormat class:**
public class KeyValueLongInputFormat extends
FileInputFormat<Text, Text> implements JobConfigurable {
private CompressionCodecFactory compressionCodecs = null;
#Override
public void configure(JobConf conf) {
compressionCodecs = new CompressionCodecFactory(conf);
}
protected boolean isSplitable(FileSystem fs, Path file) {
return compressionCodecs.getCodec(file) == null;
}
#Override
public RecordReader<Text, Text> getRecordReader(
InputSplit genericSplit, JobConf job, Reporter reporter)
throws IOException {
reporter.setStatus(genericSplit.toString());
return new KeyValueLongLineRecordReader(job, (FileSplit) genericSplit);
}
}
**Finally Mapper class:**
enter code here
public class MapperMapSideJoinLargeDatasets extends MapReduceBase implements
Mapper<Text, TupleWritable, Text, Text> {
Text txtKey = new Text("");
Text txtValue = new Text("");
#Override
public void map(Text key, TupleWritable value,
OutputCollector<Text, Text> output, Reporter reporter)
throws IOException {
if (value.toString().length() > 0) {
txtKey.set(key.toString());
String arrEmpAttributes[] = value.get(0).toString().split(",");
String arrDeptAttributes[] = value.get(1).toString().split(",");
txtValue.set(arrEmpAttributes[1].toString() + "\t"
+ arrEmpAttributes[2].toString() + "\t"
+ arrDeptAttributes[0].toString());
output.collect(txtKey, txtValue);
}
}
In the logs Map input records is 0.No output can be seen on hdfs .Someone please help me to understand this issue. Thanks
The problem lies in the Driver class where in InputFormat mentioned as KeyValueLongInputFormat.class in the strJoinStmt property which actually works for LongWritable key and Text value. Instead KeyValueTextInputFormat.class can be used when both the key and value are of Text type.As the input is comma separated file, also specify a custom delimiter as comma by setting a property in the job's configuration object as follows in the Driver class.conf.set("key.value.separator.in.input.line",",");
For Complete details please check the below example:
https://github.com/sudha-pn/CompositeInputFormat

Is there any way to delete the index of only one row in solr using solrj

I have made a java apllication which can index my last row (which is what I wabt)
But now I ask Is there any wa yo deete the index of this role! Can you give me directions how to do that or maybe simple code to change my code?
;
public class indexSolr {
private Connection conn = null;
private static HttpSolrServer server;
private Collection docs = new ArrayList();
private int _totalSql = 0;
private long _start = System.currentTimeMillis();
public static void main(String[] args) throws SolrServerException, IOException, SQLException
{ String url = "http://localhost:8983/solr/db";
indexSolr idxer = new indexSolr(url);
idxer.doSqlDocuments();
idxer.endIndexing();
}
private void doSqlDocuments() throws SQLException {
try {
Class.forName("org.postgresql.Driver");
conn = DriverManager.getConnection(
"jdbc:postgresql://localhost:5432/biz_cat",
"postgres", "pos");
java.sql.Statement st = null;
st = conn.createStatement();
ResultSet rs = st.executeQuery("select * from pl_biz order by id DESC LIMIT 1");
while (rs.next()) {
SolrInputDocument doc = new SolrInputDocument();
Integer id = rs.getInt("id");
String name = rs.getString("name");
String midname = rs.getString("midname");
String lastname = rs.getString("lastname");
String frlsname = rs.getString("frlsname");
String biz_subject = rs.getString("biz_subject");
String company_type = rs.getString("company_type");
String obshtina = rs.getString("obshtina");
String main_office_town = rs.getString("main_office_town");
String address = rs.getString("address");
String role = rs.getString("role");
String country = rs.getString("country");
String nace_code = rs.getString("nace_code");
String nace_text = rs.getString("nace_text");
String zip_code = rs.getString("zip_code");
String phone = rs.getString("phone");
String fax = rs.getString("fax");
String email = rs.getString("email");
String web = rs.getString("web");
String location = rs.getString("location");
String geohash = rs.getString("geohash");
Integer popularity = rs.getInt("popularity");
doc.addField("id", id);
doc.addField("name", name);
doc.addField("midname", midname);
doc.addField("lastname", lastname);
doc.addField("frlsname", frlsname);
doc.addField("biz_subject", biz_subject);
doc.addField("company_type", company_type);
doc.addField("obshtina", obshtina);
doc.addField("main_office_town", main_office_town);
doc.addField("address", address);
doc.addField("role", role);
doc.addField("country", country);
doc.addField("nace_code", nace_code);
doc.addField("nace_text", nace_text);
doc.addField("zip_code", zip_code);
doc.addField("phone", phone);
doc.addField("fax", fax);
doc.addField("email", email);
doc.addField("web", web);
doc.addField("location", location);
doc.addField("geohash", geohash);
doc.addField("popularity", popularity);
docs.add(doc);
++_totalSql;
if (docs.size() > 1) {
// Commit within 5 minutes.
UpdateResponse resp = server.add(docs);
System.out.println (resp);
if (resp.getStatus() != 0) {
log("Some horrible error has occurred, status is: " +
resp.getStatus());
}
docs.clear();
}
}
}
catch (Exception ex)
{
ex.printStackTrace();
}
finally {
if (conn != null) {
conn.close();
}
}
}
private void endIndexing() throws IOException, SolrServerException {
if (docs.size() > 0) { // Are there any documents left over?
server.add(docs, 300000); // Commit within 5 minutes
}
try
{
server.commit();
}
catch (Exception ex)
{
ex.printStackTrace();
}
long endTime = System.currentTimeMillis();
log("Total Time Taken: " + (endTime - _start) +
" milliseconds to index " + _totalSql +
" SQL rows" );
}
private indexSolr(String url) throws IOException, SolrServerException {
// Create a multi-threaded communications channel to the Solr server.
try {
server = new HttpSolrServer(url);
server.setSoTimeout(1000); // socket read timeout
server.setConnectionTimeout(1000);
server.setMaxRetries(1);
}
catch (Exception ex)
{
ex.printStackTrace();
}
}
}
That's all you need to do to delete the index of a row with id for example 30682 from solr
SolrServer solrServer;
String url = "http://localhost:8983/solr/db";
solrServer = new HttpSolrServer(url);
solrServer.deleteByQuery("id:30682");
solrServer.commit();
System.out.println("index deleted");
I Hope it helps someone

Shell Extension DLL - how to capture the folder path, if user clicks inside the folder empty area?

Using shell extension dll, how to capture the folder path, if user clicks inside the folder empty area?
If you're implementing a shell extension dll, then you get the path in your IShellExtInit::Initialize() method as the pidlFolder parameter.
To make sure your extension is also registered for folder backgrounds, you have to create the appropriate entries also under HKCR\Directory\Background\shellex\ContextMenuHandlers
With VC++ language please reference Winmerge souce code
http://sourceforge.net/p/winmerge/code/HEAD/tree/trunk/ShellExtension/
With C# please reference this article
http://www.codeproject.com/Articles/174369/How-to-Write-Windows-Shell-Extension-with-NET-Lang
and update some place bellow:
At FileContextMenuExt.cs file:
...............
#region Shell Extension Registration
[ComRegisterFunction()]
public static void Register(Type t)
{
try
{
ShellExtReg.RegisterShellExtContextMenuHandler(t.GUID, "Directory",
"CSShellExtContextMenuHandler.FileContextMenuExt Class");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message); // Log the error
throw; // Re-throw the exception
}
}
[ComUnregisterFunction()]
public static void Unregister(Type t)
{
try
{
ShellExtReg.UnregisterShellExtContextMenuHandler(t.GUID, "Directory");
}
catch (Exception ex)
{
Console.WriteLine(ex.Message); // Log the error
throw; // Re-throw the exception
}
}
#endregion
...............
public void Initialize(IntPtr pidlFolder, IntPtr pDataObj, IntPtr hKeyProgID)
{
if (pDataObj == IntPtr.Zero && pidlFolder == IntPtr.Zero)
{
throw new ArgumentException();
}
FORMATETC fe = new FORMATETC();
fe.cfFormat = (short)CLIPFORMAT.CF_HDROP;
fe.ptd = IntPtr.Zero;
fe.dwAspect = DVASPECT.DVASPECT_CONTENT;
fe.lindex = -1;
fe.tymed = TYMED.TYMED_HGLOBAL;
STGMEDIUM stm = new STGMEDIUM();
try
{
if (pDataObj != IntPtr.Zero)
{
// The pDataObj pointer contains the objects being acted upon. In this
// example, we get an HDROP handle for enumerating the selected files
// and folders.
IDataObject dataObject = (IDataObject)Marshal.GetObjectForIUnknown(pDataObj);
dataObject.GetData(ref fe, out stm);
// Get an HDROP handle.
IntPtr hDrop = stm.unionmember;
if (hDrop == IntPtr.Zero)
{
throw new ArgumentException();
}
// Determine how many files are involved in this operation.
uint nFiles = NativeMethods.DragQueryFile(hDrop, UInt32.MaxValue, null, 0);
// This code sample displays the custom context menu item when only
// one file is selected.
if (nFiles == 1)
{
// Get the path of the file.
StringBuilder fileName = new StringBuilder(260);
if (0 == NativeMethods.DragQueryFile(hDrop, 0, fileName,
fileName.Capacity))
{
Marshal.ThrowExceptionForHR(WinError.E_FAIL);
}
this.selectedFile = fileName.ToString();
}
else
{
Marshal.ThrowExceptionForHR(WinError.E_FAIL);
}
}
if (pidlFolder != IntPtr.Zero) {
StringBuilder folderName = new StringBuilder(260);
if (0 == NativeMethods.SHGetPathFromIDList(pidlFolder, folderName))
{
Marshal.ThrowExceptionForHR(WinError.E_FAIL);
}
this.selectedFile = folderName.ToString();
}
}
finally
{
NativeMethods.ReleaseStgMedium(ref stm);
}
}
At ShellExtLib.cs file Add folowing source:
[DllImport("shell32.dll")]
public static extern Int32 SHGetPathFromIDList(
IntPtr pidl, // Address of an item identifier list that
// specifies a file or directory location
// relative to the root of the namespace (the
// desktop).
StringBuilder pszPath); // Address of a buffer to receive the file system
And update RegisterShellExtContextMenuHandler and UnregisterShellExtContextMenuHandler function at ShellExtLib.cs file
public static void RegisterShellExtContextMenuHandler(Guid clsid,
string fileType, string friendlyName)
{
if (clsid == Guid.Empty)
{
throw new ArgumentException("clsid must not be empty");
}
if (string.IsNullOrEmpty(fileType))
{
throw new ArgumentException("fileType must not be null or empty");
}
// If fileType starts with '.', try to read the default value of the
// HKCR\<File Type> key which contains the ProgID to which the file type
// is linked.
if (fileType.StartsWith("."))
{
using (RegistryKey key = Registry.ClassesRoot.OpenSubKey(fileType))
{
if (key != null)
{
// If the key exists and its default value is not empty, use
// the ProgID as the file type.
string defaultVal = key.GetValue(null) as string;
if (!string.IsNullOrEmpty(defaultVal))
{
fileType = defaultVal;
}
}
}
}
else {
// Create the key HKCR\<File Type>\shellex\ContextMenuHandlers\{<CLSID>}.
string keyName1 = string.Format(#"{0}\Background\shellex\ContextMenuHandlers\{1}",
fileType, clsid.ToString("B"));
using (RegistryKey key = Registry.ClassesRoot.CreateSubKey(keyName1))
{
// Set the default value of the key.
if (key != null && !string.IsNullOrEmpty(friendlyName))
{
key.SetValue(null, friendlyName);
}
}
}
// Create the key HKCR\<File Type>\shellex\ContextMenuHandlers\{<CLSID>}.
string keyName = string.Format(#"{0}\shellex\ContextMenuHandlers\{1}",
fileType, clsid.ToString("B"));
using (RegistryKey key = Registry.ClassesRoot.CreateSubKey(keyName))
{
// Set the default value of the key.
if (key != null && !string.IsNullOrEmpty(friendlyName))
{
key.SetValue(null, friendlyName);
}
}
}
public static void UnregisterShellExtContextMenuHandler(Guid clsid,
string fileType)
{
if (clsid == null)
{
throw new ArgumentException("clsid must not be null");
}
if (string.IsNullOrEmpty(fileType))
{
throw new ArgumentException("fileType must not be null or empty");
}
// If fileType starts with '.', try to read the default value of the
// HKCR\<File Type> key which contains the ProgID to which the file type
// is linked.
if (fileType.StartsWith("."))
{
using (RegistryKey key = Registry.ClassesRoot.OpenSubKey(fileType))
{
if (key != null)
{
// If the key exists and its default value is not empty, use
// the ProgID as the file type.
string defaultVal = key.GetValue(null) as string;
if (!string.IsNullOrEmpty(defaultVal))
{
fileType = defaultVal;
}
}
}
}
else {
// Remove the key HKCR\<File Type>\shellex\ContextMenuHandlers\{<CLSID>}.
string keyName1 = string.Format(#"{0}\Background\shellex\ContextMenuHandlers\{1}",
fileType, clsid.ToString("B"));
Registry.ClassesRoot.DeleteSubKeyTree(keyName1, false);
}
// Remove the key HKCR\<File Type>\shellex\ContextMenuHandlers\{<CLSID>}.
string keyName = string.Format(#"{0}\shellex\ContextMenuHandlers\{1}",
fileType, clsid.ToString("B"));
Registry.ClassesRoot.DeleteSubKeyTree(keyName, false);
}