[moriarty] r314 committed - Merge down (again) of branch with mirror from url changes.
mori...@googlecode.com
Author: jhi...@gmail.com
Date: Tue Aug 2 01:29:25 2011
Log: Merge down (again) of branch with mirror from url changes.
http://code.google.com/p/moriarty/source/detail?r=314
Modified:
/trunk
/trunk/changeset.class.php
/trunk/graph.class.php
/trunk/simplegraph.class.php
/trunk/store.class.php
/trunk/tests/graph.test.php
/trunk/tests/simplegraph.test.php
=======================================
--- /trunk/changeset.class.php Tue Aug 2 01:13:40 2011
+++ /trunk/changeset.class.php Tue Aug 2 01:29:25 2011
@@ -50,6 +50,7 @@
var $_index = array();
function __construct($a = '') {
+ parent::__construct();
$this->a = $a;
/* parse the before and after graphs if necessary*/
foreach(array('before','after', 'before_rdfxml', 'after_rdfxml') as
$rdf){
=======================================
--- /trunk/graph.class.php Tue Aug 2 01:13:40 2011
+++ /trunk/graph.class.php Tue Aug 2 01:29:25 2011
@@ -125,6 +125,31 @@
$request->set_body( $turtle );
return $request->execute();
}
+
+ function
submit_ntriples_in_batches_from_file($filename,$no_of_lines=500,
$stop_on_failure=true) {
+ $responses = array();
+ $pointer = fopen($filename, 'r');
+ $batch = '';
+ $lineCount=0;
+ while($line = fgets($pointer)){
+ $batch.=$line;
+ $lineCount++;
+ if($lineCount==$no_of_lines){
+ $response = $this->submit_turtle($batch);
+ $responses[] = $response;
+ if($response->is_success()===false AND $stop_on_failure){
+ return $responses;
+ } else {
+ $lineCount=0;
+ $batch='';
+ }
+ }
+ }
+ if(!empty($batch)){
+ $responses[]=$this->submit_turtle($batch);
+ }
+ return $responses;
+ }
/**
* Obtain the graph's bounded description of a given resource
=======================================
--- /trunk/simplegraph.class.php Tue Aug 2 01:13:40 2011
+++ /trunk/simplegraph.class.php Tue Aug 2 01:29:25 2011
@@ -1372,5 +1372,38 @@
return $values;
}
+ /** Skolemise Bnodes
+ * replace bnodes in the graph with URIs
+ * @param urispace
+ **/
+
+ public function skolemise_bnodes($urispace)
+ {
+ $bnodes = $this->get_bnodes();
+ $skolemised_bnodes = array();
+ foreach($bnodes as $no => $bnode){
+ $uri = $urispace.'id-'.++$no;
+ $this->replace_resource($bnode, $uri);
+ $skolemised_bnodes[$bnode] = $uri;
+ }
+ return $skolemised_bnodes;
+ }
+
+ public function get_bnodes()
+ {
+ $bnodes = array();
+ $index = $this->get_index();
+ foreach($index as $s => $ps){
+ if(strpos($s,'_:')===0) $bnodes[]=$s;
+ foreach($ps as $p => $os){
+ foreach($os as $o){
+ if($o['type']=='bnode'){
+ $bnodes[]=$o['value'];
+ }
+ }
+ }
+ }
+ return array_unique($bnodes);
+ }
}
?>
=======================================
--- /trunk/store.class.php Tue Aug 2 01:13:40 2011
+++ /trunk/store.class.php Tue Aug 2 01:29:25 2011
@@ -12,7 +12,7 @@
require_once MORIARTY_DIR. 'augmentservice.class.php';
require_once MORIARTY_DIR. 'oaiservice.class.php';
require_once MORIARTY_DIR. 'httprequest.class.php';
-
+require_once MORIARTY_DIR. 'changeset.class.php';
/**
* Represents a platform store.
*/
@@ -203,6 +203,91 @@
{
return $this->get_contentbox()->save_item($content, $content_type);
}
+
+
+
+ /**
+ * mirror_from_url
+ *
+ * @return array of responses from http requests, and overall success
status
+ * @author Keith Alexander
+ **/
+ function mirror_from_url($url, $rdf_content=false)
+ {
+
+ $return = array(
+ 'get_page' => false,
+ 'get_copy' => false,
+ 'put_page' => false,
+ 'update_data' => false,
+ 'success' => false,
+ );
+
+ if (empty( $this->request_factory) ) {
+ $this->request_factory = new HttpRequestFactory();
+ }
+
+ $last_cached_page_uri = $this->get_contentbox()->uri.'/mirrors/'.$url;
+
+ if(!$rdf_content){
+
+ $web_page_request = $this->request_factory->make('GET', $url);
+
$web_page_request->set_accept('application/rdf+xml;q=0.8,text/turtle;q=0.9,*/*;q=0.1');
+ $web_page_response = $web_page_request->execute();
+ $return['get_page'] = $web_page_response;
+ $web_page_content = $web_page_response->body;
+ } else {
+ $web_page_content = $rdf_content;
+ $return['rdf_content'] = $rdf_content;
+ }
+ if($rdf_content OR $web_page_response->is_success() ){
+
+ $newGraph = new SimpleGraph();
+ $newGraph->add_rdf($web_page_content);
+ $newGraph->add_resource_triple($url, OPEN_LASTCACHEDPAGE,
$last_cached_page_uri);
+ $newGraph->skolemise_bnodes($last_cached_page_uri.'/');
+ $after = $newGraph->get_index();
+ # get previous copy if it exists
+ $cached_page_request = $this->request_factory->make('GET',
$last_cached_page_uri, $this->credentials);
+ $cached_page_response = $cached_page_request->execute();
+ $return['get_copy'] = $cached_page_response;
+ if($cached_page_response->status_code == '200'){
+ $before = json_decode($cached_page_response->body, true);
+ } else if( $cached_page_response->status_code == '404' ) {
+ $before = false;
+ } else {
+ return $return;
+ }
+ $put_page_request = $this->request_factory->make('PUT',
$last_cached_page_uri, $this->credentials);
+ $put_page_request->set_body($newGraph->to_json());
+ $put_page_request->set_content_type('application/json');
+ $put_page_response = $put_page_request->execute();
+ $return['put_page'] = $put_page_response;
+
+ if(!$put_page_response->is_success()){
+ return $return;
+ }
+ # build new changeset
+
+ $Changeset = new ChangeSet(array('before' => $before, 'after' =>
$after));
+
+ if($Changeset->has_changes()){
+ $return['update_data'] =
$this->get_metabox()->apply_changeset($Changeset);
+ if($return['update_data']->is_success()){
+ $return['success'] = true;
+ }
+ return $return;
+ } else {
+ $return['success'] = true;
+ return $return;
+ }
+
+
+ } else {
+
+ return $return;
+ }
+ }
}
?>
=======================================
--- /trunk/tests/graph.test.php Tue Aug 2 01:13:40 2011
+++ /trunk/tests/graph.test.php Tue Aug 2 01:29:25 2011
@@ -352,6 +352,27 @@
$response = $g->submit_turtle( $this->_turtle_doc );
$this->assertTrue( in_array('Accept: */*',
$fake_request->get_headers() ) );
}
+
+ function test_submit_ntriples_from_file_in_batches(){
+ $response = new HttpResponse('202');
+ $graph = $this->getMock('Graph', array('submit_turtle'),
array(),'',false );
+ $graph->expects($this->exactly(10))
+ ->method('submit_turtle')
+ ->will($this->returnValue($response));
+ $filename = dirname(__FILE__).'/documents/10-ntriples.nt';
+ $result = $graph->submit_ntriples_in_batches_from_file($filename,1);
+ $this->assertEquals(count($result), 10);
+ }
+ function test_submit_ntriples_from_file_in_batches_stop_on_failure(){
+ $response = new HttpResponse('500');
+ $graph = $this->getMock('Graph', array('submit_turtle'),
array(),'',false );
+ $graph->expects($this->exactly(1))
+ ->method('submit_turtle')
+ ->will($this->returnValue($response));
+ $filename = dirname(__FILE__).'/documents/10-ntriples.nt';
+ $result =
$graph->submit_ntriples_in_batches_from_file($filename,1,true);
+ $this->assertEquals(count($result), 1);
+ }
}
=======================================
--- /trunk/tests/simplegraph.test.php Tue Aug 2 01:13:40 2011
+++ /trunk/tests/simplegraph.test.php Tue Aug 2 01:29:25 2011
@@ -1139,5 +1139,99 @@
}
+
+ public function test_skolemise_bnodes(){
+
+ $input = array(
+ '_:a' => array(
+ RDFS_LABEL => array(
+ array(
+ 'value' => 'A Bnode',
+ 'type' => 'literal',
+ ),
+ array(
+ 'value' => '_:b',
+ 'type' => 'bnode',
+ ),
+ ),
+ ),
+
+ '_:b' => array(
+ RDFS_LABEL => array(
+ array(
+ 'type' => 'literal',
+ 'value' => 'bnode B',
+ )
+ )
+ )
+
+ );
+
+ $expected_output = array(
+ 'http://example.org/document/id-1' => array(
+ RDFS_LABEL => array(
+ array(
+ 'value' => 'A Bnode',
+ 'type' => 'literal',
+ ),
+ array(
+ 'value' => 'http://example.org/document/id-2',
+ 'type' => 'uri',
+ ),
+
+ ),
+ ),
+ 'http://example.org/document/id-2' => array(
+ RDFS_LABEL => array(
+ array(
+ 'type' => 'literal',
+ 'value' => 'bnode B',
+ )
+ )
+ )
+ );
+
+ $graph = new SimpleGraph($input);
+ $graph->skolemise_bnodes('http://example.org/document/');
+ $output = $graph->get_index();
+ $this->assertEquals($expected_output, $output, "bnodes in the graph
should be replaced with URIs");
+
+
+ }
+
+
+ function test_graph_pattern_is_unchanged_by_replace_resource(){
+
+ }
+
+ function
test_number_of_resources_remains_constant_after_skolemise_bnodes(){
+ $graph = new
SimpleGraph(file_get_contents(dirname(__FILE__).'/documents/ckan-ds.ttl'));
+ $index = $graph->get_index();
+ $before = count($graph->get_subjects());
+ $graph->skolemise_bnodes('http://example.com/test/');
+ $after = count($graph->get_subjects());
+ $this->assertEquals($before, $after, "skolemise_bnodes shouldn't
reduce the number of resources");
+ }
+
+ function test_get_bnodes(){
+
+ $input = array(
+ '_:a' => array(
+ RDFS_SEEALSO => array(
+ array(
+ 'value' => '_:b',
+ 'type' => 'bnode',
+ ),
+ ),
+ ),
+
+ );
+
+
+ $graph = new SimpleGraph($input);
+ $actual = $graph->get_bnodes();
+ $expected = array('_:a', '_:b');
+ $this->assertEquals($expected, $actual, "get_bnodes() should return
bnodes in subject and object positions");
+ }
}
?>