Rick

Rick
Rick

Wednesday, June 16, 2010

I was messing around with the google collection API. It has the ability to do functional-like programming.

When using inner classes where you should use closures, one looks forward to Java with closures.

import java.util.Collection;

import java.util.Set;

import org.junit.Test;

import com.google.common.base.Function;

import com.google.common.base.Predicate;

import com.google.common.collect.Collections2;

import com.google.common.collect.ImmutableSet;

public class GoogleCollectionTest {

public static class Employee {

private String firstName;

private String lastName;

private String phone;

public Employee(String firstName, String lastName, String phone) {

this.firstName = firstName;

this.lastName = lastName;

this.phone = phone;

}

public Employee() {

}

public String getFirstName() {

return firstName;

}

public String getLastName() {

return lastName;

}

public String getPhone() {

return phone;

}

public String toString() {

return new StringBuilder(80).append(firstName).append(' ').append(lastName).toString();

}

}

public static class Phone {

private String areaCode;

private String prefix;

private String number;

public Phone() {

}

public Phone(String areaCode, String prefix, String number) {

this.areaCode = areaCode;

this.prefix = prefix;

this.number = number;

}

public String getAreaCode() {

return areaCode;

}

public String getPrefix() {

return prefix;

}

public String getNumber() {

return number;

}

@Override

public String toString() {

return new StringBuilder(50).append('(').append(areaCode).append(')')

.append(prefix).append('-').append(number).toString();

}

}

public static Employee employee(String firstName, String lastName, String phone) {

return new Employee(firstName, lastName, phone);

}

public static Set employees (Employee... arguments) {

return ImmutableSet.of(arguments);

}

public static class OnlyTucsonEmployees implements Predicate{

public boolean apply(Employee input) {

if (input.getPhone().startsWith("520")) {

return true;

}

return false;

}

}

private static final OnlyTucsonEmployees tucsonEmployeesPredicate = newOnlyTucsonEmployees();

public static class EmployeeToPhone implements Function {

@Override

public Phone apply(Employee from) {

String[] phoneComps = from.phone.split("-");

return new Phone(phoneComps[0], phoneComps[1], phoneComps[2]);

}

}

private static final EmployeeToPhone employeeToPhoneFunction = new EmployeeToPhone();

@Test

public void Test() {

Set employees = employees(

employee("Rick", "Hightower", "520-555-1212"),

employee("Noah", "Hightower", "520-555-1212"),

employee("Ryan", "Hightower", "520-555-1212"),

employee("Lucas", "Hightower", "520-555-1212"),

employee("Whitney", "Hightower", "520-555-1212"),

employee("Stephane", "Loonadi", "614-888-1515")

);

Collection tucsonEmployees = Collections2.filter(employees,tucsonEmployeesPredicate);

System.out.println(tucsonEmployees);

Collection phoneNumbers = Collections2.transform(employees,employeeToPhoneFunction);

System.out.println(phoneNumbers);

Collection tucsonPhoneNumbers = Collections2.transform(Collections2.filter(employees, tucsonEmployeesPredicate),employeeToPhoneFunction);

System.out.println(tucsonPhoneNumbers);

}

}

Friday, May 7, 2010

This script has evolved quite a bit since I changed it to work with our new REST API.
It is a handy little script for functionally testing REST JSON end points and then storing the results as text files in SVN for later comparison with new test runs (and run against other instances).



#!/usr/bin/env groovy
verbose = false
execute = true
test = true
autogen = false
count = 0 //total number of tests
passed = 0 //number of tests that passed
generated = 0 //number of tests that failed
jsonExceptions = 0 //number of JSON expceptions
nullResults = 0
xmlFaults=0
ioExceptions=0
host=null
port=null
webapp=null
cwd = new File(".") //current working directory
inputDir = new File(cwd, "input") //Read test files from the inputDir


testsAllowNull = new File(cwd, "conf/expectNulls.conf").text.split(",") as List

println "Tests that allow nulls $testsAllowNull"


if (!processArgs(args)) {
System.exit(0)
}

def processArgs(args) {
def cli = new CliBuilder(usage: "testScript.groovy -[hoagpv]")

cli.with {
h longOpt: "help", "Show usage information"
o longOpt: "host", args: 1, "Set host name"
g longOpt: "autogen", "turn on autogen"
a longOpt: "webapp", args: 1, "set the webapplication URI, i.e., palmcs or ac"
p longOpt: "port", args:1, "Set the port for the URL"
v longOpt: "verbose", "Verbose mode"
i longOpt: "inputDir", args:1, "Specify the input dir"
}

options = cli.parse(args)
if (!options) System.exit(0)

if (options.o) host = options.o
if (options.p) port = options.p
if (options.a) webapp = options.a
if (options.i) inputDir = new File(options.i)
verbose = options.v
autogen = options.g

if (verbose) {
println """
host $host
port $port
webapp URI $webapp
autogen $autogen
verbose $verbose
input dir $inputDir
"""
}

if (options.h) {
cli.usage()
if (verbose) {
println """
The testScript.groovy is a test script that tests JSON payloads against JSON endpoints.
It essentially sends a JSON payload (from the input dir) and then receives a JSON payload.
Then it compares the JSON payload received with a corresponding reference file (should match).
It has some nice features that dump everything out to the file system (in formatted JSON) so you can use
Beyond Compare and/or Araxis to see the exact difference between the actual and the expected.

To get this message run:
./testScript.groovy -v -h

To run the script in verbose mode:
./testScript.groovy -v

To run the script in verbose mode:
./testScript.groovy -v

To run the script against a different host, port and webapp URI:
./testScript.groovy -onewhost -p9999 -anewwebapp

The above would run against
http://newhost:9999/newwebapp/services/deviceJ/addUserRating

You can specify host or (host and port) or (host and port and webapp).

To run the script against just a different host (same port, same webapp URI):

./testScript.groovy -onewhost

The above would run against
http://newhost:8101/ac/services/deviceJ/addUserRating

CWD is the current working directory.

Input files are searched for in CWD/input and have the following format:

http://localhost:8101/ac/services/deviceJ/appDeleted
{ "InAppDeleted" : {
"accountTokenInfo":{"token":"assds","deviceId":"asd","email":"test@a.com","carrier":""},
"publicApplicationId":"com.palm.app.ace",
"version" : "1.0"
}
}

The first line is the URL (which can be changed via -o, -p and -a options).
The rest of the file is the JSON payload.

The files to compare the JSON payload from the server against are under CWD/compare.

If the file comparison fails, the payload from the server is stored in CWD/failed_compare so you can use
Beyond Compare or Araxis Merge or their ilk to see the differences easily.

When you first run the tests against a new server, you may need/want to create some reference JSON payloads to compare.
You can do this with the autogenerate option as follows:

./testScript.groovy -olocalhost -p8101 -apalmcsext -g

The above would create a new set of comparison JSON payloads for host localhost, port 8101,
and webapp name palmcsext.
It is up to you to look at the file contents and see if they are correct.
Once you get a set of correct files you can use them for integration testing.

"""
} else {
println "To see example usage run -h with -v (help and verbose)"
}
return false
}

return true

}



/* Read all test scripts that end in .tst in the input directory. */
inputDir.eachFileMatch(~/^.*\.tst$/) {File inFile->
processFile(inFile)
}

def processURL(url) {
url = url.toURL()
boolean urlEndsWithSlash=url.path.endsWith("/")

if (host || port || webapp) {
if (host && !port && !webapp) {
url = "http://${host}:${url.port}${url.path}${urlEndsWithSlash?"/":""}${url.query?"?${url.query}":""}"
} else if (host && port && !webapp) {
url = "http://${host}:${port}${url.path}${urlEndsWithSlash?"/":""}${url.query?"?${url.query}":""}"
} else if (host && port && webapp) {
restOfPathAfterWebApp = (url.path.split("/") as List)[2..-1].join("/")
url = "http://${host}:${port}/${webapp}/${restOfPathAfterWebApp}${urlEndsWithSlash?"/":""}${url.query?"?${url.query}":""}"
}
}

println "URL WE ARE GOING TO RUN ${url}"
return (urlEndsWithSlash ? url + "/" : url).toURL()
}

/* Process the input file. */
def processFile(File testFile) {
println "Running test $testFile.name"
lines = testFile.readLines()
body = new StringBuilder(512)
url = ""
method = ""
headers = [:]
format = true

lines.eachWithIndex{line, index ->
if (index==0) {
url = line
} else {

if (line.startsWith("HEADER# ")) {
nv = line.split("# ")[1].split(":=")
headers[nv[0]]=nv[1]
} else if (line.startsWith("METHOD# ")) {
method = line.split("# ")[1]
} else if (line.startsWith("FORMAT# ")) {
sFormat = line.split("# ")[1]
if (sFormat.equalsIgnoreCase("off")) {
format = false
}
}
body.append(line)
body.append("\n")
}
}
body = body.toString()

if (verbose) println "$url \n $body"
url = processURL(url)

/* Create the input file name which is like ./compare/localhost_8101_ac/getAppDetail.tst.compare. */
compareDir = new File(cwd, "compare")
compareDir = new File(compareDir, "${url.host}_${url.port}_${url.path.split('/')[1]}_for_${inputDir.name}")
compareDir.mkdirs()
compareFile = new File(compareDir,
"${testFile.name}.compare")

if (verbose) println "Compare file $compareFile"

if (execute) {
execute(testFile, url, body, compareFile, method, headers, format)
}
}


/** Executes a JSON test based on a URL and a body. */
def execute(testFile, url, body, compareFile, method, headers, format) {
/* Execute the JSON payload body to the given URL. */
results = executeMethod(testFile, url, body, method, headers, format)

if (verbose) {
println "RESULTS:"
println results
}

/* Only run tests if flag is set (so you can use verbose to see what you would run) */
if(test) {
test(testFile, compareFile, results)
}

autogen(compareFile, results)
}


/** Automatically generates the output to compare with from the results from the server. */
def autogen(compareFile, results) {
if (verbose) println "--- AUTOGEN is $autogen $compareFile ${results!=null}--- "
if (autogen && !compareFile.exists() && results) {

compareFile.write(results)
generated++
}
}

/** Run the actual test/validation/assertions against the results. */
def test(testFile, compareFile, results) {
count++ //increment the number of tests run
allowNull = testFile.name in testsAllowNull
/* Check to see if the results are null unless they are allowed to be null. */
if (!results && !allowNull) {
nullResults++;
println "The test $testFile.name returned null"
/* Check to see if the results are a JSON Exception, a JSON exception may not be an error
* we need to improve this to check against a list of tests we expect JSON exceptions from.
*/
} else if (results && results.contains("JSONException")) {
jsonExceptions++
println "The test $testFile.name threw a JSONException"
/* Check to see if we got an XMLFault. An XML fault is always an error. */
} else if(results && results.contains("XMLFault")) {
xmlFaults++
println "The test $testFile.name threw an XMLFault"
/* Check to see if the results match the comparison file on record. */
} else if (results && compareFile.exists()) {
/* Remove all whitespace and see if the string from the server matches the reference file. */
if (compareFile.text.split().join(' ') .equals(results.split().join(' '))) {
passed++
/* For really big payloads, Java string comparison fails, so
* execute the cmp from the command line and see if the files match.
*/
} else {
if (!testUsingCMPFromCommandLine(compareFile, results)) {
//if (compareFile.text.substring(0,150).equals(results.substring(0,150)) ){
// passed++;
//}
}
}
} else if (! results && allowNull) {
passed++
}
}

def testUsingCMPFromCommandLine(compareFile, results) {
failedDir = new File(cwd, "failed_compare")
failedDir = new File(failedDir, "${url.host}_${url.port}_${url.path.split('/')[1]}_for_${inputDir.name}")
failedDir.mkdirs()

failedCompareFile = new File(failedDir, compareFile.name)

compareCommand = "cmp $compareFile $failedCompareFile"

if (results) {
failedCompareFile.write(results)
cmpProcess = compareCommand.execute()
pText = cmpProcess.text
if (verbose) println pText
cmpProcess.waitFor()
if (cmpProcess.exitValue() == 0){
passed++
}
else {
println "The test failed cmp compare $compareCommand"
return false
}
}

return true
}

def executeMethod(testFile, url, body, method, headers, format) {

if (verbose) {
println """
headers = $headers
method = $method
format = $format
"""
}
httpConnection = url.openConnection()
httpConnection.connectTimeout = 180000
httpConnection.requestMethod = method
httpConnection.doOutput = true
httpConnection.doInput = true
httpConnection.useCaches = false
httpConnection.allowUserInteraction = false
httpConnection.followRedirects = false
httpConnection.setRequestProperty("Content-type", "application/json;charset=UTF-8")
if (headers) {
headers.each{key, value ->
if (verbose) println "Setting Header $key $value"
httpConnection.setRequestProperty(key, value)
}
}
if (method=="POST" || method == "PUT") {
httpConnection.outputStream.write(body.getBytes("UTF-8"))
httpConnection.outputStream.flush()
httpConnection.outputStream.close()
}
def json = null
def responseHeaders = []
def responseCode = -1
def responseMessage = "nothing"
try {

responseCode = httpConnection.responseCode
responseMessage = httpConnection.responseMessage
if (verbose) println "response code $httpConnection.responseCode"
if (responseCode==200) {
json = httpConnection.inputStream.readLines().join("\n")
} else {
if (httpConnection.errorStream) {
json = httpConnection.errorStream.readLines().join("\n")
}
}
for (int index in 0..10) {
value = httpConnection.getHeaderField(index)
if (!value) break
if (httpConnection.getHeaderFieldKey(index) == "Date") continue
responseHeaders.add("HEADER: ${httpConnection.getHeaderFieldKey(index)}:=${value}")
}

} catch (IOException iex) {
println "$testFile.name was unable to connect: $iex.message"
ioExceptions++
return null
} catch(Exception ex) {
ex.printStackTrace()
println "Unexpected error"
}

/* Add new line characters after every curly braket '}' and comma ','. */
if (json && format) {
json = json.replace("}", "}\n").replace(",", ",\n")
}
return """
#RESPONSE CODE: $responseCode
#RESPONSE MESSAGE: $responseMessage
#HEADERS START:
${responseHeaders.join('\n')}
#HEADERS END:
$json
END
"""
}

println "Test count = ${count}, passed = ${passed}, generated = ${generated}, IO error = $ioExceptions"
println "JSON Exceptions = ${jsonExceptions}, null results = ${nullResults}, XML Faults=${xmlFaults}"



Saturday, April 3, 2010

Very Useful

I found this to be very very useful!

Thanks!

If you have a legacy db and you want to use Roo, this is a good guide.

in reference to: Life in the startup lane » Tutorial for using Spring Roo with an existing database (view on Google Sidewiki)

Tuesday, March 30, 2010

Optimizing JPA calls (using JPA, ROO, Hibernate, Maven, Spring and Tomcat)

I created a proof of concept for our new REST based JSON API.

I used ROO, Spring 3 and JPA.

(This is just a POC and not a real app at this point.)

One of the issues is the number of queries we make for a relatively simple access call to get a list of the top level categories and a list of categories.

The current implementation of the POC does not use any caching.

NOTE:
If you are using ROO, Spring, JPA Maven and/or the Maven Tomcat plugin, I think this will help you setup caching and log4j. It will also help you optimize you object retrieval with JPA. Thus, I post it here where people can see it, find it, and use it. If it helps, you good. Feel free to provide feedback.

As an exercise in architecture design, I wanted to show what it would take to optimize the queries and to add 2nd level caching. Then we can decide if it makes sense for future JSON APIs.

{
"outer" : {
"categoryList" : {
"categoryItem" : [ {
"id" : 1,
"count":2,
"name" : "Games"
}, {
"id" : 2,
"count":4
"name" : "Utils"
} ]
}
}
}
We need to look up the categories by spoken language.

As JPA goes, the Category domain object is fairly complex, as it uses a self join through a join table. But JPA does a good job of hiding the complexity and making representing this object fairly straight forward as follows:

@Entity
@RooJavaBean
@RooToString
@Table(name = "ad_categories")
@RooEntity(versionField = "", finders = { "findCategorysByCommonLanguage" })
@JsonIgnoreProperties( { "parent", "commonLanguage", "subcategories" })
public class Category {

private String name;

@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@Column(name = "id")
private Long id;

@ManyToOne(targetEntity = Category.class)
@JoinTable(name = "ad_category_hierarchy", joinColumns =
@JoinColumn(name = "childid"), inverseJoinColumns = @JoinColumn(name = "parentid"))
private Category parent;

@ManyToOne(targetEntity = CommonLanguage.class)
@JoinColumn(name = "languageid")
private CommonLanguage commonLanguage;

@Formula("(select count(*) from ad_category_hierarchy ah join " +
"ad_categories ac on ac.id = ah.childid where ah.parentid = id)")
private int count;

@ManyToMany(cascade = CascadeType.ALL)
@JoinTable(name = "ad_category_hierarchy", joinColumns = @JoinColumn(name = "parentid"), inverseJoinColumns = @JoinColumn(name = "childid"))
private Set subcategories = new HashSet();

public static Query findCategoriesByCommonLanguageAndParent(
CommonLanguage commonLanguage, Category parent) {
if (commonLanguage == null)
throw new IllegalArgumentException(
"The commonLanguage argument is required");

EntityManager em = Category.entityManager();
Query q = null;

if (parent == null) {
q = em
.createQuery("SELECT Category FROM Category AS category WHERE category.commonLanguage = :commonLanguage AND category.parent is null");
} else {
q = em
.createQuery("SELECT Category FROM Category AS category WHERE category.commonLanguage = :commonLanguage AND category.parent = :parent");

}
q.setParameter("commonLanguage", commonLanguage);
if (parent != null) {
q.setParameter("parent", parent);
}
return q;
}
}
The above is a fairly naked, first attempt of using JPA to manage a category domain object.

Here is the controller we use to return a JSON response to the client:

@RooWebScaffold(path = "category", automaticallyMaintainView = true, formBackingObject = Category.class)
@RequestMapping("/category/**")
@Controller
public class CategoryController {

/**
* List Categories is used to return a JSON payload of cateogories.
*
*
* @param locale the locale in the request (currently we only use the language from the locale)
* @param parentId If the parentId is 0 then just look up the top level categories, else return categories for that parent
* @param response gives up control over what response code we send to the client.
* @return
*/
@SuppressWarnings("unchecked")
@RequestMapping(value = "/category/listCategories/locale/{locale}/category/{parentId}", method = RequestMethod.GET)
public @ResponseBody List listCategories(@PathVariable String locale,
@PathVariable Long parentId,
HttpServletResponse response) {

/* Locale is required. */
if (locale == null || locale.trim().length()==0) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
//TODO log this
return null;
}

/* If the parentId is 0, then set it to null, otherwise look up the parent category. */
Category parent = parentId==0 ? null : Category.findCategory(parentId);

/* Extract the language part of the locale to look up the common language. */
String language = locale.split("_")[0];
CommonLanguage commonLanguage = (CommonLanguage) CommonLanguage.findCommonLanguagesByCode(language).getSingleResult();

/* Look up the list of categories. */
List categories = Category.findCategoriesByCommonLanguageAndParent(commonLanguage, parent).getResultList();
return categories;
}

}
Focus mainly on this part of the code:

Category parent = parentId==0 ? null : Category.findCategory(parentId);

String language = locale.split("_")[0];
CommonLanguage commonLanguage = (CommonLanguage) CommonLanguage.findCommonLanguagesByCode(language).getSingleResult();

List categories = Category.findCategoriesByCommonLanguageAndParent(commonLanguage, parent).getResultList();
return categories;
It is fairly easy to instrument Hibernate to show you exactly what SQL it is producing to return a call as follows:

persistence.xml



When hitting the above JSON end point, we get the following SQL queries.

URL:

http://localhost:8080/appcatalog/category/listCategories/locale/en_US/category/0

Output:

[
{
"name": "Other",
"id": 1,
"count": 0
},
{
"name": "Books",
"id": 2,
"count": 14
},
{
"name": "Business",
"id": 3,
"count": 11
},
{
"name": "School",
"id": 4,
"count": 11
},
{
"name": "Games",
"id": 5,
"count": 9
},
{
"name": "Finance",
"id": 6,
"count": 9
},
{
"name": "Food",
"id": 7,
"count": 5
},
...
]
The above uses the following SQL:

Mar 30, 2010 11:45:53 AM org.apache.catalina.core.ApplicationContext log
INFO: Initializing Spring FrameworkServlet 'appcatalog'
Hibernate:
/* SELECT
CommonLanguage
FROM
CommonLanguage AS commonlanguage
WHERE
commonlanguage.code = :code */ select
commonlang0_.id as id0_,
commonlang0_.code as code0_,
commonlang0_.description as descript3_0_,
commonlang0_.name as name0_
from
common_language commonlang0_
where
commonlang0_.code=?
Hibernate:
/* SELECT
Category
FROM
Category AS category
WHERE
category.commonLanguage = :commonLanguage
AND category.parent is null */ select
category0_.id as id2_,
category0_.languageid as languageid2_,
category0_.name as name2_,
category0_1_.parentid as parentid3_,
(select
count(*)
from
ad_category_hierarchy ah
join
ad_categories ac
on ac.id = ah.childid
where
ah.parentid = category0_.id) as formula0_
from
ad_categories category0_
left outer join
ad_category_hierarchy category0_1_
on category0_.id=category0_1_.childid
where
category0_.languageid=?
and (
category0_1_.parentid is null
)
If we specify a parent id category as follows:

URL: http://localhost:8080/appcatalog/category/listCategories/locale/en_US/category/15

Then we get the following:

Hibernate:
/* load com.mycompany.appcatalog.domain.Category */ select
category0_.id as id2_2_,
category0_.languageid as languageid2_2_,
category0_.name as name2_2_,
category0_1_.parentid as parentid3_2_,
(select
count(*)
from
ad_category_hierarchy ah
join
ad_categories ac
on ac.id = ah.childid
where
ah.parentid = category0_.id) as formula0_2_,
commonlang1_.id as id0_0_,
commonlang1_.code as code0_0_,
commonlang1_.description as descript3_0_0_,
commonlang1_.name as name0_0_,
category2_.id as id2_1_,
category2_.languageid as languageid2_1_,
category2_.name as name2_1_,
category2_1_.parentid as parentid3_1_,
(select
count(*)
from
ad_category_hierarchy ah
join
ad_categories ac
on ac.id = ah.childid
where
ah.parentid = category2_.id) as formula0_1_
from
ad_categories category0_
left outer join
ad_category_hierarchy category0_1_
on category0_.id=category0_1_.childid
left outer join
common_language commonlang1_
on category0_.languageid=commonlang1_.id
left outer join
ad_categories category2_
on category0_1_.parentid=category2_.id
left outer join
ad_category_hierarchy category2_1_
on category2_.id=category2_1_.childid
where
category0_.id=?
Hibernate:
/* SELECT
CommonLanguage
FROM
CommonLanguage AS commonlanguage
WHERE
commonlanguage.code = :code */ select
commonlang0_.id as id0_,
commonlang0_.code as code0_,
commonlang0_.description as descript3_0_,
commonlang0_.name as name0_
from
common_language commonlang0_
where
commonlang0_.code=?
Hibernate:
/* SELECT
Category
FROM
Category AS category
WHERE
category.commonLanguage = :commonLanguage
AND category.parent = :parent */ select
category0_.id as id2_,
category0_.languageid as languageid2_,
category0_.name as name2_,
category0_1_.parentid as parentid3_,
(select
count(*)
from
ad_category_hierarchy ah
join
ad_categories ac
on ac.id = ah.childid
where
ah.parentid = category0_.id) as formula0_
from
ad_categories category0_
left outer join
ad_category_hierarchy category0_1_
on category0_.id=category0_1_.childid
where
category0_.languageid=?
and category0_1_.parentid=?
Ok so in the first case (top level cat) we do the following:

Select on common language
Select on category with inner select and an outer join (quite expensive in terms of db calls)
In the second case things get pretty exciting, we do the following:

Select on category with an inner join and count them 1, 2, 3, 4 outer joins
Select on Common language
Select on category with inner select and outer join
What immediate comes to mind is that I can get rid of the calls to load commonlanguage by rolling the lang code into the category query as follows:

public static Query findCategoriesByLanguageCodeAndParent(
String languageCode, Category parent) {
if (languageCode == null)
throw new IllegalArgumentException(
"The commonLanguage argument is required");

EntityManager em = Category.entityManager();
Query q = null;

if (parent == null) {
q = em
.createQuery("SELECT Category FROM Category AS category WHERE category.commonLanguage.code = :languageCode AND category.parent is null");
} else {
q = em
.createQuery("SELECT Category FROM Category AS category WHERE category.commonLanguage.code = :languageCode AND category.parent = :parent");

}
q.setParameter("languageCode", languageCode);
if (parent != null) {
q.setParameter("parent", parent);
}
return q;
}

...

public @ResponseBody List listCategories(@PathVariable String locale,
@PathVariable Long parentId,
HttpServletResponse response) {

/* Locale is required. */
if (locale == null || locale.trim().length()==0) {
response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
//TODO log this
return null;
}

/* If the parentId is 0, then set it to null, otherwise look up the parent category. */
Category parent = parentId==0 ? null : Category.findCategory(parentId);

/* Extract the language part of the locale to look up the category by language code. */
String languageCode = locale.split("_")[0];

/* Look up the list of categories. */
List categories = Category.findCategoriesByLanguageCodeAndParent(languageCode, parent).getResultList();
return categories;
}
My first knee jerk reaction was to add a second level cache but after looking at the SQL, the solution became very clear. (Simple refactoring..)

Ok... all of the above did was switch the order. Hibernate still loaded the common language. We need to make it lazy load it on demand.

I went ahead and marked all of the relationships as lazy as we can override the lazy load behavior with the Criteria API or with JPA QL later if needed.

Here is an updated set of queries:

First case (load top level cats):

URL: http://localhost:8080/appcatalog/category/listCategories/locale/en_US/category/0

Hibernate:
/* SELECT
Category
FROM
Category AS category
WHERE
category.commonLanguage.code = :languageCode
AND category.parent is null */ select
category0_.id as id2_,
category0_.languageid as languageid2_,
category0_.name as name2_,
category0_1_.parentid as parentid3_,
(select
count(*)
from
ad_category_hierarchy ah
join
ad_categories ac
on ac.id = ah.childid
where
ah.parentid = category0_.id) as formula0_
from
ad_categories category0_
left outer join
ad_category_hierarchy category0_1_
on category0_.id=category0_1_.childid,
common_language commonlang1_
where
category0_.languageid=commonlang1_.id
and commonlang1_.code=?
and (
category0_1_.parentid is null
)
We got the hits to the database go down to 1.

Now for the second case:

URL: http://localhost:8080/appcatalog/category/listCategories/locale/en_US/category/2

Hibernate:
/* load com.mycompany.appcatalog.domain.Category */ select
category0_.id as id2_0_,
category0_.languageid as languageid2_0_,
category0_.name as name2_0_,
category0_1_.parentid as parentid3_0_,
(select
count(*)
from
ad_category_hierarchy ah
join
ad_categories ac
on ac.id = ah.childid
where
ah.parentid = category0_.id) as formula0_0_
from
ad_categories category0_
left outer join
ad_category_hierarchy category0_1_
on category0_.id=category0_1_.childid
where
category0_.id=?
Hibernate:
/* SELECT
Category
FROM
Category AS category
WHERE
category.commonLanguage.code = :languageCode
AND category.parent = :parent */ select
category0_.id as id2_,
category0_.languageid as languageid2_,
category0_.name as name2_,
category0_1_.parentid as parentid3_,
(select
count(*)
from
ad_category_hierarchy ah
join
ad_categories ac
on ac.id = ah.childid
where
ah.parentid = category0_.id) as formula0_
from
ad_categories category0_
left outer join
ad_category_hierarchy category0_1_
on category0_.id=category0_1_.childid,
common_language commonlang1_
where
category0_.languageid=commonlang1_.id
and commonlang1_.code=?
and category0_1_.parentid=?
So the db hit count goes down to 2 from 3 and we are no longer doing a five way join.

We are still loading the top level category and then loading its children. It would behoove us to setup an object cache for categories due to the natural fan out of a hierarchal list of taxonomy like an app catalog.

After we setup the cache, we should get the hits to the db to 1 per use case.

Here are the new lazy fields for JPA:

public class Category {

private String name;

@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@Column(name = "id")
private Long id;

@ManyToOne(targetEntity = Category.class, fetch=FetchType.LAZY)
@JoinTable(name = "ad_category_hierarchy", joinColumns =
@JoinColumn(name = "childid"), inverseJoinColumns = @JoinColumn(name = "parentid"))
private Category parent;

@ManyToOne(targetEntity = CommonLanguage.class, fetch=FetchType.LAZY)
@JoinColumn(name = "languageid")
private CommonLanguage commonLanguage;

@Formula("(select count(*) from ad_category_hierarchy ah join " +
"ad_categories ac on ac.id = ah.childid where ah.parentid = id)")
private int count;

@ManyToMany(cascade = CascadeType.ALL, fetch=FetchType.LAZY)
@JoinTable(name = "ad_category_hierarchy", joinColumns = @JoinColumn(name = "parentid"), inverseJoinColumns = @JoinColumn(name = "childid"))
private Set subcategories = new HashSet();
Ok before we setup the second level cache, we need to turn on log4j logging so we can see that it is setup.

We use the maven tomcat plugin. Tomcat uses log4j internally so it is difficult to get tomcat to respect your log4j setting.

To get it to respect them, you have to go nuclear and setup log4j at the JVM level in order to do that you need to setup the MAVEN_OPTS environment variable, e.g.,

cat.sh
rickhightower$ cat env.sh
export JAVA_HOME=/System/Library/Frameworks/JavaVM.framework/Versions/1.6.0/Home
export CATALINA_HOME=`pwd`/tomcat

export MAVEN_OPTS="-Xmx1024m -XX:MaxPermSize=512m -Dlog4j.configuration=file:/Users/rickhightower/proto/application-catalog/src/main/resources/META-INF/spring/log4j.properties"
Then in the log4j.properties turn on Hibernate cache logging.

log4j.properties
log4j.rootLogger=error, stdout, R

log4j.appender.stdout=org.apache.log4j.ConsoleAppender
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout

# Print the date in ISO 8601 format
log4j.appender.stdout.layout.ConversionPattern=%d [%t] %-5p %c - %m%n

log4j.appender.R=org.apache.log4j.RollingFileAppender
log4j.appender.R.File=application.log

log4j.appender.R.MaxFileSize=100KB
# Keep one backup file
log4j.appender.R.MaxBackupIndex=1

log4j.appender.R.layout=org.apache.log4j.PatternLayout
log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n

log4j.logger.org.hibernate.cache=DEBUG
When it comes to debugging Hibernate and Spring issues you cannot fly blind. You have to setup logging. If you don't know how, then you don't know how to use Spring and Hibernate.

Then you need to tell Hibernate about ehcache configuration:

persistence.xml






Notice we setup hibernate.cache.use_second_level_cache to true. Then we configure a provider and a location for the cache setup file.

Then you need to setup a ehcache file to configure the cache as follows:

ehcache-ac.xml



name="com.mycompany.appcatalog.domain.Category"
maxElementsInMemory="1000"
eternal="false"
timeToIdleSeconds="0"
timeToLiveSeconds="3600"
overflowToDisk="false"
/>

maxElementsInMemory="10000"
eternal="false"
timeToIdleSeconds="120"
timeToLiveSeconds="120"
overflowToDisk="true"
maxElementsOnDisk="10000000"
diskPersistent="false"
diskExpiryThreadIntervalSeconds="120"
memoryStoreEvictionPolicy="LRU"
/>

Then lastly, we need to enable caching for the particular object see @Cache annotation below:

Category.java
@Entity
@Cache(usage = CacheConcurrencyStrategy.READ_ONLY )
public class Category {

private String name;

@Id
@GeneratedValue(strategy = GenerationType.AUTO)
@Column(name = "id")
private Long id;
Kafka and Cassandra support, training for AWS EC2 Cassandra 3.0 Training