[virgil] r143 committed - Finished release assembly.
virgil.apach...@codespot.com
Author: boneill42
Date: Mon Jan 16 20:39:41 2012
Log: Finished release assembly.
http://code.google.com/a/apache-extras.org/p/virgil/source/detail?r=143
Added:
/trunk/release/assembly/example
/trunk/release/assembly/example/insert-data.sh
/trunk/release/assembly/example/remove-data.sh
/trunk/release/assembly/example/run-mapreduce.sh
/trunk/release/assembly/example/wordcount.rb
/trunk/release/pom.xml
Modified:
/trunk/pom.xml
/trunk/release/assembly/distribution.xml
/trunk/release/assembly/mapreduce/jars/virgil-mapreduce-hdeploy.jar
/trunk/server/pom.xml
/trunk/server/src/main/java/org/apache/virgil/resource/MapReduceResource.java
=======================================
--- /dev/null
+++ /trunk/release/assembly/example/insert-data.sh Mon Jan 16 20:39:41 2012
@@ -0,0 +1,22 @@
+# Create the keyspace for input
+curl -X PUT http://localhost:8080/virgil/data/dummy/
+
+# Create the column family for output
+curl -X PUT http://localhost:8080/virgil/data/dummy/book
+
+# Create the keyspace for output
+curl -X PUT http://localhost:8080/virgil/data/stats/
+
+# Create the column family for output
+curl -X PUT http://localhost:8080/virgil/data/stats/word_counts
+
+# Insert some rows
+curl -X PUT http://localhost:8080/virgil/data/dummy/book/brian
-d "{\"page1\":\"I like cold beverages\",\"page2\":\"I like
dogs\",\"page3\":\"I like brew\",\"page4\":\"Sausages are good\"}"
+
+curl -X PUT http://localhost:8080/virgil/data/dummy/book/lisa
-d "{\"page1\":\"I like red wine\",\"page2\":\"I like
strawberries\",\"page3\":\"I dislike brussel sprouts\",\"page4\":\"Life is
good.\"}"
+
+curl -X PUT http://localhost:8080/virgil/data/dummy/book/collin
-d "{\"page1\":\"I like pepperoni\"}"
+
+curl -X PUT http://localhost:8080/virgil/data/dummy/book/owen
-d "{\"page1\":\"I like watermelon\"}"
+
+curl -X PUT http://localhost:8080/virgil/data/dummy/book/maya
-d "{\"page1\":\"dog food is good\"}"
=======================================
--- /dev/null
+++ /trunk/release/assembly/example/remove-data.sh Mon Jan 16 20:39:41 2012
@@ -0,0 +1,3 @@
+# Delete the keyspace for input
+curl -X DELETE http://localhost:8080/virgil/data/dummy/
+curl -X DELETE http://localhost:8080/virgil/data/stats/
=======================================
--- /dev/null
+++ /trunk/release/assembly/example/run-mapreduce.sh Mon Jan 16 20:39:41
2012
@@ -0,0 +1,1 @@
+curl -X POST
http://localhost:8080/virgil/job?jobName=wordcount\&inputKeyspace=dummy\&inputColumnFamily=book\&outputKeyspace=stats\&outputColumnFamily=word_counts
--data-binary @wordcount.rb
=======================================
--- /dev/null
+++ /trunk/release/assembly/example/wordcount.rb Mon Jan 16 20:39:41 2012
@@ -0,0 +1,22 @@
+def map(rowKey, columns)
+ result = []
+ columns.each do |column_name, value|
+ words = value.split
+ words.each do |word|
+ result << [word, "1"]
+ end
+ end
+ return result
+end
+
+def reduce(key, values)
+ rows = {}
+ total = 0
+ columns = {}
+ values.each do |value|
+ total += value.to_i
+ end
+ columns["count"] = total.to_s
+ rows[key] = columns
+ return rows
+end
=======================================
--- /dev/null
+++ /trunk/release/pom.xml Mon Jan 16 20:39:41 2012
@@ -0,0 +1,43 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.virgil</groupId>
+ <artifactId>virgil-parent</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>virgil</artifactId>
+ <packaging>pom</packaging>
+ <name> Virgil : Release </name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.virgil</groupId>
+ <artifactId>virgil-mapreduce</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.virgil</groupId>
+ <artifactId>virgil-server</artifactId>
+ <version>0.10.0-SNAPSHOT</version>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2.1</version>
+ <configuration>
+ <descriptors>
+ <descriptor>assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
=======================================
--- /trunk/pom.xml Mon Jan 16 19:05:29 2012
+++ /trunk/pom.xml Mon Jan 16 20:39:41 2012
@@ -11,8 +11,9 @@
<modules>
<module>server</module>
<module>mapreduce</module>
+ <module>release</module>
</modules>
-
+
<dependencies>
<dependency>
<groupId>log4j</groupId>
@@ -38,7 +39,22 @@
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-all</artifactId>
<version>1.0.7</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>guava</artifactId>
+ <groupId>com.google.guava</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
+
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -47,28 +63,28 @@
</dependency>
</dependencies>
- <build>
- <plugins>
- <plugin>
-
<artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.6</source>
- <target>1.6</target>
- <showWarnings>true</showWarnings>
- </configuration>
- <version>2.3.2</version>
- </plugin>
- <plugin>
-
<artifactId>maven-assembly-plugin</artifactId>
- <version>2.2.1</version>
- <configuration>
- <descriptors>
-
<descriptor>assembly/distribution.xml</descriptor>
- </descriptors>
- </configuration>
- </plugin>
- </plugins>
- </build>
+ <build>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ <showWarnings>true</showWarnings>
+ </configuration>
+ <version>2.3.2</version>
+ </plugin>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.2.1</version>
+ <configuration>
+ <descriptors>
+ <descriptor>assembly/distribution.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
</project>
=======================================
--- /trunk/release/assembly/distribution.xml Mon Jan 16 19:05:29 2012
+++ /trunk/release/assembly/distribution.xml Mon Jan 16 20:39:41 2012
@@ -27,17 +27,17 @@
<outputDirectory>conf</outputDirectory>
</fileSet>
<fileSet>
- <directory>${basedir}/mapreduce/jars</directory>
+ <directory>${basedir}/assembly/example</directory>
+ <outputDirectory>example</outputDirectory>
+ </fileSet>
+ <fileSet>
+ <directory>${basedir}/assembly/mapreduce/jars</directory>
<outputDirectory>mapreduce/jars</outputDirectory>
</fileSet>
<fileSet>
- <directory>${basedir}/mapreduce/conf</directory>
+ <directory>${basedir}/assembly/mapreduce/conf</directory>
<outputDirectory>mapreduce/conf</outputDirectory>
</fileSet>
- <fileSet>
- <directory>${basedir}/server/src/main/webapp</directory>
- <outputDirectory>www</outputDirectory>
- </fileSet>
</fileSets>
<dependencySets>
<dependencySet>
=======================================
--- /trunk/release/assembly/mapreduce/jars/virgil-mapreduce-hdeploy.jar Mon
Jan 16 19:05:29 2012
+++ /trunk/release/assembly/mapreduce/jars/virgil-mapreduce-hdeploy.jar Mon
Jan 16 20:39:41 2012
File is too large to display a diff.
=======================================
--- /trunk/server/pom.xml Mon Jan 16 19:05:29 2012
+++ /trunk/server/pom.xml Mon Jan 16 20:39:41 2012
@@ -13,10 +13,6 @@
<name>Virgil : Server</name>
<url>http://maven.apache.org</url>
- <properties>
- <cassandra.data.dir>/var/lib</cassandra.data.dir>
- </properties>
-
<dependencies>
<dependency>
<groupId>org.apache.virgil</groupId>
@@ -35,13 +31,6 @@
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
</dependency>
-
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.7</version>
- <scope>test</scope>
- </dependency>
</dependencies>
<build>
@@ -51,26 +40,6 @@
<directory>${basedir}/src/main/resources</directory>
</resource>
</resources>
- <plugins>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.6</source>
- <target>1.6</target>
- <showWarnings>true</showWarnings>
- </configuration>
- <version>2.3.2</version>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.2.1</version>
- <configuration>
- <descriptors>
- <descriptor>src/main/assembly/distribution.xml</descriptor>
- </descriptors>
- </configuration>
- </plugin>
- </plugins>
</build>
</project>
=======================================
---
/trunk/server/src/main/java/org/apache/virgil/resource/MapReduceResource.java
Mon Jan 16 18:38:55 2012
+++
/trunk/server/src/main/java/org/apache/virgil/resource/MapReduceResource.java
Mon Jan 16 20:39:41 2012
@@ -31,8 +31,7 @@
@QueryParam("inputKeyspace") String inputKeyspace,
@QueryParam("inputColumnFamily") String inputColumnFamily,
@QueryParam("outputKeyspace") String outputKeyspace,
- @QueryParam("outputColumnFamily") String outputColumnFamily,
@QueryParam("remote") boolean remote,
- String source) throws Exception {
+ @QueryParam("outputColumnFamily") String outputColumnFamily,
String source) throws Throwable {
if (inputKeyspace == null)
throw new RuntimeException("Must supply inputKeyspace.");
if (inputColumnFamily == null)
@@ -48,24 +47,14 @@
logger.debug(" <-- Output : Keyspace [" + outputKeyspace
+ "], ColumnFamily [" + outputColumnFamily + "]");
}
- if (remote) {
-
+ if (this.virgilService.getStorage().isEmbedded()) {
+ logger.debug("Running in embedded mode.");
+ JobSpawner.spawnLocal(jobName,
this.getCassandraStorage().getHost(), this.getCassandraStorage().getPort(),
+ inputKeyspace, inputColumnFamily, outputKeyspace,
outputColumnFamily, source, params);
} else {
- try {
- if (this.virgilService.getStorage().isEmbedded()) {
- JobSpawner.spawnLocal(jobName,
this.getCassandraStorage().getHost(), this.getCassandraStorage()
- .getPort(), inputKeyspace, inputColumnFamily,
outputKeyspace, outputColumnFamily, source,
- params);
- } else {
- JobSpawner.spawnRemote(jobName,
this.getCassandraStorage().getHost(), this.getCassandraStorage()
- .getPort(), inputKeyspace, inputColumnFamily,
outputKeyspace, outputColumnFamily, source,
- params);
-
- }
- } catch (Throwable e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
+ logger.debug("Spawning job remotely.");
+ JobSpawner.spawnRemote(jobName,
this.getCassandraStorage().getHost(), this.getCassandraStorage().getPort(),
+ inputKeyspace, inputColumnFamily, outputKeyspace,
outputColumnFamily, source, params);
}
}