[svn openjsan.org] [466] Much better RSS feeds.

[ca...@geeknest.com]

Revision: 466
Author: cwest
Date: 2005-10-25 13:27:15 -0400 (Tue, 25 Oct 2005)

Log Message:
-----------
Much better RSS feeds.
Resolved indexing problems.
Decoupled processor and indexer.
Fixed small warnings and bugs.

Modified Paths:
--------------
openjsan/trunk/bin/existing-processor
openjsan/trunk/lib/JSAN/Incoming.pm
openjsan/trunk/templates/src/news/index.html

Added Paths:
-----------
openjsan/trunk/bin/site-level-processor
Modified: openjsan/trunk/bin/existing-processor
===================================================================
--- openjsan/trunk/bin/existing-processor 2005-09-13 14:21:39 UTC (rev 465)
+++ openjsan/trunk/bin/existing-processor 2005-10-25 17:27:15 UTC (rev 466)
@@ -14,7 +14,7 @@
->in( "$htdocs/dist" );

foreach my $in_file ( @dists ) {
-warn "processing $in_file...\n";
+ warn ">>>>>>>>> $in_file...\n";
my $incoming = JSAN::Incoming->new(file => $in_file);
$incoming->process_existing;
}

Added: openjsan/trunk/bin/site-level-processor
===================================================================
--- openjsan/trunk/bin/site-level-processor 2005-09-13 14:21:39 UTC (rev 465)
+++ openjsan/trunk/bin/site-level-processor 2005-10-25 17:27:15 UTC (rev 466)
@@ -0,0 +1,10 @@
+#!/usr/local/bin/perl
+use strict;
+use warnings;
+
+use lib qw[lib ../lib];
+use JSAN::Incoming;
+
+my $incoming = JSAN::Incoming->new;
+$incoming->process_site_level;
+


Property changes on: openjsan/trunk/bin/site-level-processor
___________________________________________________________________
Name: svn:executable
+ *

Modified: openjsan/trunk/lib/JSAN/Incoming.pm
===================================================================
--- openjsan/trunk/lib/JSAN/Incoming.pm 2005-09-13 14:21:39 UTC (rev 465)
+++ openjsan/trunk/lib/JSAN/Incoming.pm 2005-10-25 17:27:15 UTC (rev 466)
@@ -64,6 +64,7 @@
_setup_archive
_write_dist
_write_src
+ _set_dist_description
_find_js_libs
];

@@ -85,6 +86,12 @@
_create_author_index
_create_js_dist_version_index
_import_tags
+ ];
+}
+
+sub process_site_level {
+ my ($self) = @_;
+ $self->$_() for qw[
_create_master_yaml_index
_create_recent_rss
_create_tag_pages
@@ -106,6 +113,7 @@
_setup_history
_setup_archive
_write_dist
+ _set_dist_description
_find_js_libs
];

@@ -127,9 +135,6 @@
_create_author_index
_create_js_dist_version_index
_import_tags
- _create_master_yaml_index
- _create_recent_rss
- _create_tag_pages
];
}

@@ -163,16 +168,12 @@
sub _import_tags {
my ($self) = @_;

- my $meta = $self->_get_meta();
+ my $meta = ($self->_get_meta())[0];
return unless $meta;
- return unless $meta->{keywords};
+ return unless $meta->keywords;

my $dist = $self->dist;
- $self->dist_description( $meta->{abstract}) if $meta->{abstrat};
- $self->dist_name( $meta->{name} ) if $meta->{name}
- $self->dist_version( $meta->{version} ) if $meta->{version};
-
- foreach my $tag ( @{$meta->{keywords}} ) {
+ foreach my $tag ( @{$meta->keywords} ) {
my $tag = $self->jsan->data->tag->find_or_create({name => $tag});
$self->jsan->data->distribution_tag->find_or_create({
distribution => $dist,
@@ -183,6 +184,10 @@

sub _create_recent_rss {
my ($self) = @_;
+ my @dists = sort { $b->cdate <=> $a->cdate }
+ grep { $_->cdate } # only processed distributions
+ $self->jsan->data->distribution->retrieve_all;
+
my $rss = XML::RSS->new( version => '1.0' );

$rss->channel(
@@ -213,23 +218,19 @@
);

my $dist_class = $self->jsan->data->distribution;
- for my $dist ($dist_class->retrieve_from_sql(q{
- id IN (
- SELECT max(id)
- FROM distributions
- GROUP BY name
- ORDER by max(id) DESC
- LIMIT 30
- )
+ foreach my $dist ($dist_class->retrieve_from_sql(q{
+ 1 = 1 order by cdate desc limit 30
})) {
my $path = $dist->author->hash . '/'
. join('/',split /\./,$dist->name) . '/' . $dist->version;
+ my $readme_path = $dist->author->hash . '/'
+ . $dist->name . '-' . $dist->version;
$rss->add_item(
title => join('-', $dist->name, $dist->version),
description => '<pre>' . encode_entities(
- $self->_load_readme($path) || $dist->description
+ $self->_load_readme($readme_path) || $dist->description || ''
) . '</pre>',
- link => "http://openjsan.org/doc/$path";
+ 'link' => "http://openjsan.org/doc/$path",
dc => {
date => Time::Piece->new($dist->cdate)->datetime,
},
@@ -240,9 +241,12 @@

sub _load_readme {
my $self = shift;
- my $path = $self->htdocs . '/' . shift;
+ my $path = $self->htdocs . '/src/' . shift;
my $readme = $self->_find_file('readme', $path) or return;
- return < io $readme;
+ my $text = io($self->htdocs . '/' . $readme)->slurp;
+ # cheap way to get rid of HTML constructs
+ $text =~ s/<(\w)/&lt;$1/g;
+ return $text;
}


@@ -271,6 +275,7 @@
foreach my $dist ( @dists ) {
my $name = $dist->name;

+ next unless $dist->name; # unprocessed distribution in database
$index->{distributions}->{$name}->{doc} = '/doc/' . $author->hash . '/' . join('/',split /\./,$dist->name);

$index->{distributions}->{$name}->{releases} = []
@@ -322,6 +327,8 @@
my $type = undef;
if (-e $meta_file) {
$self->meta_file_type('yml');
+ $self->_fix_meta_yaml_file($meta_file);
+ #warn "-------------- $meta_file\n";
eval { $meta_text = Module::META::YAML->read($meta_file) };
warn $@ if $@;
return '' if $@;
@@ -338,7 +345,19 @@
return ($meta_text, $type);
}

+sub _fix_meta_yaml_file {
+ my ($self, $yaml_file) = @_;

+ my $yaml = io($yaml_file)->slurp;
+
+ # recommend instead of recommends
+ $yaml =~ s/recommend:/recommends:/g;
+
+ # new line at end of file
+ $yaml .= "\n" unless $yaml =~ /\n\Z/;
+ io($yaml_file) < $yaml;
+}
+
sub _create_js_dist_version_index {
my ($self) = @_;
$self->_build_dist_version_html(
@@ -556,19 +575,38 @@
sub _update_dist_info {
my ($self) = @_;
my @parts = split /-/, $self->src_dist_name;
- my $dist = $self->dist;
- $dist->version($self->dist_version || pop @parts);
- $dist->name($self->dist_name || join '.', @parts);
- $disg->description($self->dist_description);
- $dist->checksum(md5_hex do {
- open DIST, $self->html_dist;
- join '', <DIST>
- });
- $dist->cdate(time) unless $dist->cdate;
- $dist->srcdir($self->src_dist_name);
- $dist->update;
+ my $version = pop @parts;
+ $self->dist->version($version);
+ $self->dist->name(join '.', @parts);
+ my $checksum = $self->_get_dist_checksum;
+ $self->dist->checksum($checksum);
+ $self->dist->cdate(time) unless $self->dist->cdate;
+ $self->dist->srcdir($self->src_dist_name);
+ $self->dist->description($self->dist_description);
+ $self->dist->update;
}

+sub _set_dist_description {
+ my ($self) = @_;
+
+ my $meta = ($self->_get_meta())[0];
+ return unless $meta;
+
+ $self->dist_description( $meta->{abstract}) if $meta->{abstract};
+ $self->dist_name( $meta->{name} ) if $meta->{name};
+ $self->dist_version( $meta->{version} ) if $meta->{version};
+
+ $self->_update_dist_info;
+}
+
+sub _get_dist_checksum {
+ my ($self) = @_;
+ my $dist_file = $self->html_dist;
+ my $dist_data = io($dist_file)->slurp;
+ my $checksum = md5_hex($dist_data);
+ return $checksum;
+}
+
sub _set_src_dist {
my ($self) = @_;
my $srcdistnam = fileparse(
@@ -684,9 +722,10 @@
sub _find_file {
my ($self, $fn, $source) = @_;

- my ($file) = File::Find::Rule->new->file->name(qr/$fn/i)->in(join '/', $self->htdocs, $source);
- return unless $file;
my $htdocs = $self->htdocs;
+ $source =~ s/$htdocs\///;
+ my ($file) = File::Find::Rule->new->file->name(qr/$fn/i)->maxdepth(1)->in(join '/', $htdocs, $source);
+ return undef unless $file;
$file =~ s/$htdocs\///;
return $file;
}
@@ -694,14 +733,12 @@
my ($self, %args) = @_;
my $tt = $self->tt;

- my %dists = (
- map { $_[1]->name} => $_ }
- sort { $a->[1] <=> $b->[1] }
- map { [ qv($_->version) => $_ ] }
- $self->jsan->data->distribution->search(
- author => $args{author},
- )
- );
+ my %dists;
+ my @dists = sort { qv($a->version) <=> qv($b->version) }
+ $self->jsan->data->distribution->search(
+ author => $args{author},
+ );
+ $dists{$_->name} = $_ for @dists;

my $htdocs = $self->htdocs;
$tt->process('author_index', {

Modified: openjsan/trunk/templates/src/news/index.html
===================================================================
--- openjsan/trunk/templates/src/news/index.html 2005-09-13 14:21:39 UTC (rev 465)
+++ openjsan/trunk/templates/src/news/index.html 2005-10-25 17:27:15 UTC (rev 466)
@@ -2,6 +2,25 @@

<h1>News</h1>

+<h2>Update - 2005-10-25</h2>
+
+<ul>
+ <li>Greatly improved recent uploads RSS feed. Thanks in huge part to David Wheeler.</li>
+ <li>Fixed the processing and indexing system. There were a lot of little bugs.</li>
+ <li>Decoupled the processing and indexing system.</li>
+ <li>System is now more stable, and close to automating the processing of incoming distributions.</li>
+ <li>Next steps are notifying an author when processing is complete, and notifying the author if processing failed.</li>
+</ul>
+
+<h2>Update - 2005-08-31</h2>
+
+<ul>
+ <li>Fixed the <a href="/tag">tag cloud</a>.</li>
+ <li>Began working at <a href="http://socialtext.com">SocialText</a> on Monday</li>
+</ul>
+
+<!-- __END__ -->
+
<h2>Update - 2005-08-18</h2>

<ul>
@@ -19,8 +38,6 @@
huge changes in my life that I'll write about later.</li>
</ul>

-<!-- __END__ -->
-
<h2>Update - 2005-07-26</h2>

<ul>