#!/usr/bin/perl -w our $ID = q$Id: spin-rss,v 1.25 2012/12/23 05:01:26 eagle Exp $; # # spin-rss -- Generate RSS and thread from a feed description file. # # Copyright 2008 Russ Allbery # # This program is free software; you may redistribute it and/or modify it # under the same terms as Perl itself. ############################################################################## # Modules and declarations ############################################################################## require 5.006; use strict; use Date::Parse qw(str2time); use Getopt::Long qw(GetOptions); use POSIX qw(strftime); ############################################################################## # Utility functions ############################################################################## # Returns the intersection of two arrays as a list. sub intersect { my ($a, $b) = @_; my %a = map { $_ => 1 } @$a; return grep { $a{$_} } @$b; } # Construct an absolute URL from a relative URL and a base URL. sub absolute_url { my ($url, $base) = @_; $base =~ s,[^/]+$,,; if ($url =~ /^[a-z]+:/) { return $url; } elsif ($url =~ m,^/,) { $base =~ s,^(https?://[^/]+).*,$1,; return "$base$url"; } else { while ($url =~ s,^\.\./+,,) { $base =~ s,[^/]+/+$,,; } return "$base$url"; } } # Construct a relative URL from an absolute URL and a base URL. sub relative_url { my ($url, $base) = @_; return $url unless $base; $base =~ s,^(https?://[^/]+/)/*,,; my $host = $1; if (!$host || index ($url, $host) != 0) { return $url; } $url =~ s,^\Q$host\E/*,,; my @base = split (m,/+,, $base); while ($url and @base) { my $segment = shift @base; unless ($url =~ s,^\Q$segment\E/*,,) { return ('../' x (@base + 1)) . $url; } } return ('../' x @base) . $url; } ############################################################################## # Parsing ############################################################################## # Parse a change file. Save the metadata into the provided hash reference and # the changes into the provided array reference. Each element of the array # will be a hash with keys title, date, link, and description. sub parse_changes { my ($file, $metadata, $changes) = @_; open (CHANGES, '<', $file) or die "$0: cannot open $file: $!\n"; local $_; my ($last, @blocks); push (@blocks, {}); my $current = $blocks[0]; while () { if (/^\s*$/) { push (@blocks, {}); $current = $blocks[$#blocks]; undef $last; } elsif (/^(\S+):[ \t]+(.+)/) { my ($key, $value) = ($1, $2); die "$0: cannot parse $file at line $.\n" unless $value; $value =~ s/\s+$//; $last = lc $key; $current->{$last} = $value; } elsif (/^(\S+):/) { $last = lc $1; $current->{$last} ||= ''; } elsif (/^\s/) { die "$0: cannot parse $file at line $.\n" unless $last; my $value = $_; $value =~ s/^\s//; $value = "\n" if $value =~ /^\.\s*\n/; if ($current->{$last} and $current->{$last} !~ /\n\z/) { $current->{$last} .= "\n"; } $current->{$last} .= $value; } } close CHANGES; pop @blocks unless $last; %$metadata = %{ shift @blocks }; $metadata->{recent} = 15 unless defined $metadata->{recent}; my %guids; for my $block (@blocks) { $block->{date} = str2time ($block->{date}) or die "$0: cannot parse date $block->{date}\n"; if ($block->{link} && $block->{link} !~ /^http/) { if ($block->{link} eq '/') { $block->{link} = $metadata->{base}; } else { $block->{link} = $metadata->{base} . $block->{link}; } } unless ($block->{guid}) { my $guid; if ($block->{journal} || $block->{review}) { $guid = $block->{link}; } else { $guid = $block->{date}; } die "$0: duplicate GUID for entry $guid\n" if $guids{$guid}; $block->{guid} = $guid; } if ($block->{tags}) { $block->{tags} = [ split (' ', $block->{tags}) ]; } else { $block->{tags} = []; } push (@{ $block->{tags} }, 'review') if $block->{review}; } @$changes = @blocks; } ############################################################################## # RSS output ############################################################################## # Escape a string for XML. sub xml_escape { my ($string) = @_; $string =~ s/&/&/g; $string =~ s//>/g; return $string; } # Format a journal post into HTML for inclusion in an RSS feed. sub rss_journal { my ($file) = @_; my @page = `spin '$file'`; if ($? != 0) { die "$0: spin of $file failed with status ", ($? >> 8), "\n"; } shift @page while (@page and $page[0] !~ /

/); shift @page; shift @page while (@page and $page[0] =~ /^\s*$/); pop @page while (@page and $page[$#page] !~ /

/); pop @page; pop @page while (@page and $page[$#page] =~ /^\s*$/); return join ('', @page) . "\n"; } # Format a review into HTML for inclusion in an RSS feed. Takes the path to # the review thread file and the metadata for this feed. sub rss_review { my ($file, $metadata) = @_; my $dir = $file; $dir =~ s%/+[^/]*$%%; my $class = ($dir =~ /magazines/) ? 'magazines' : 'books'; my $base = $metadata->{base} . ($metadata->{base} =~ m%/$% ? '' : '/') . 'reviews'; my @page = `spin '$file'`; if ($? != 0) { die "$0: spin of $file failed with status ", ($? >> 8), "\n"; } my ($title, $author); while (@page and $page[0] !~ //) { if ($page[0] =~ m{

(.*)

}) { $title = $1; } elsif ($page[0] =~ m{

(.*)

}) { $author = $1; } shift @page; } die "$0: cannot find title and author in $file" unless ($title && $author); pop @page while (@page and $page[$#page] !~ /

/); my ($buy, $ebook); for my $i (0 .. $#page) { if ($page[$i] =~ /

/) { $ebook = $i; } if ($page[$i] =~ /

/) { $buy = $i; last; } } splice (@page, $buy, 2) if $buy; splice (@page, $ebook, 4) if $ebook; my $page = join ('', @page); $page =~ s/^\s*]+>/

/mg; $page =~ s/^\s*]+>/
/mg; $page =~ s{

}{}; $page =~ s/
//; $page =~ s/

/

/; $page =~ s{(.*?)} {$1}sg; $page = "

Review: $title, $author

\n\n" . $page; return $page . "\n"; } # Print out the RSS version of the changes information given a file to which # to print it, the file name, the metadata resulting RSS file, and a reference # to the array of entries. Various things are still hard-coded here. Use the # date of the last change as and the current time as # ; it's not completely clear to me that this is correct. sub rss_output { my ($file, $name, $metadata, $entries) = @_; $name =~ s,.*/,,; my $format = '%a, %d %b %Y %H:%M:%S %z'; my $date = strftime ($format, localtime); my $last; if (@$entries) { $last = strftime ($format, localtime $entries->[0]{date}); } else { $last = $date; } my $version = (split (' ', $ID))[2]; print $file <<"EOC"; $metadata->{title} $metadata->{base} $metadata->{description} $metadata->{language} $last $date spin-rss $version EOC if ($metadata->{'rss-base'}) { print $file qq( \n); } print $file "\n"; for my $entry (@$entries) { my $date = strftime ($format, localtime $entry->{date}); my $title = xml_escape ($entry->{title}); my $description; if ($entry->{description}) { $description = xml_escape ($entry->{description}); $description =~ s/^/ /mg; $description =~ s,^(\s*),$1

,; $description =~ s,\n*\z,

\n,; } elsif ($entry->{journal}) { $description = rss_journal ($entry->{journal}); } elsif ($entry->{review}) { $description = rss_review ($entry->{review}, $metadata); } $description =~ s{(<(?:a href|img src)=\")(?!http:)([./\w][^\"]+)\"} {$1 . absolute_url ($2, $entry->{link}) . '"'}ge; my $perma = ($entry->{guid} =~ /^http/) ? '' : ' isPermaLink="false"'; print $file " \n"; print $file " $title\n"; print $file " $entry->{link}\n"; print $file " \n"; print $file " $date\n"; print $file " $entry->{guid}\n"; print $file " \n"; } print $file "
\n"; print $file "
\n"; } ############################################################################## # Thread output ############################################################################## # Print out the thread version of the recent changes list, given a file to # which to print it, the metadata, and a reference to the array of entries. sub thread_output { my ($file, $metadata, $entries) = @_; if ($metadata->{'thread-prefix'}) { print $file $metadata->{'thread-prefix'}, "\n"; } else { print $file "\\heading[Recent Changes][indent]\n\n"; print $file "\\h1[Recent Changes]\n\n"; } my $last; for my $entry (@$entries) { my $month = strftime ('%B %Y', localtime $entry->{date}); if (not $last or $month ne $last) { print $file "\\h2[$month]\n\n"; $last = $month; } my $date = strftime ('%Y-%m-%d', localtime $entry->{date}); print $file "\\desc[$date \\entity[mdash]\n"; print $file " \\link[$entry->{link}]\n"; print $file " [$entry->{title}]][\n"; my $description = $entry->{description}; $description =~ s/^/ /mg; $description =~ s/\\/\\\\/g; print $file $description; print $file "]\n\n"; } print $file "\\signature\n"; } ############################################################################## # Index output ############################################################################## # Translate the thread of a journal entry for inclusion in an index page. # Also takes the full URL for the permanent link to the entry page. Returns # the thread. sub index_journal { my ($file, $url) = @_; open (IN, '<', $file) or die "$0: cannot open $file: $!\n"; local $_; while () { last if /\\h1/; } my $text = ; $text = '' if $text =~ /^\s*$/; while () { last if /^\\date/; $text .= $_; } close IN; return $text; } # Translate the thread of a book review for inclusion into an index page. # Also takes the full URL for the permanent link to the entry page. Returns # the thread. sub index_review { my ($file, $url) = @_; open (IN, '<', $file) or die "$0: cannot open $file: $!\n"; local $_; my ($title, $author); while () { my $char = '(?:[^\]\\\\]|\\\\entity\[\S+\])'; if (/\\(header|edited)\s*\[($char+)\]\s*$/) { $_ .= ; } if (/\\(header|edited)\s*\[($char+)\]\s*\[($char+)\]/) { ($title, $author) = ($2, $3); $author .= ' (ed.)' if ($1 eq 'edited'); last; } } unless (defined $author) { die "$0: cannot parse review file $file\n"; } my $text; if ($file =~ m,/magazines/,) { $text = "Review: \\cite[$title], $author\n\n"; } else { $text = "Review: \\cite[$title], by $author\n\n"; } $text .= "\\table[][\n"; while () { last if /^\\div\(review\)\[/; my $char = '(?:[^\]\\\\]|\\\\entity\[\S+\])'; if (/^\s*\\data\[($char+)\]\s*\[($char+)\]/) { $text .= " \\tablerow[$1][$2]\n"; } } $text .= "]\n\n"; while () { last if /^\\done/; s/\\story\[\d+\]/\\strong/g; s/^\\rating\s*\[([^\]]+)\]/Rating: $1 out of 10/; $text .= $_; } close IN; return $text; } # Print out the index version of the recent changes list, given a file to # which to print it, the metadata, and a reference to the array of entries. sub index_output { my ($file, $metadata, $entries) = @_; if ($metadata->{'index-prefix'}) { print $file $metadata->{'index-prefix'}, "\n"; } my $last; for my $entry (@$entries) { my $date = strftime ('%Y-%m-%d %H:%M', localtime $entry->{date}); my $day = $date; $day =~ s/ .*//; print $file "\\h2[$day: $entry->{title}]\n\n"; my $text; if ($entry->{journal}) { $text = index_journal ($entry->{journal}, $entry->{link}); } elsif ($entry->{review}) { $text = index_review ($entry->{review}, $entry->{link}); } $text =~ s{(\\(?:link|image)\s*)\[([^\]]+)\]} {"${1}[" . absolute_url ($2, $entry->{link}) . ']'}ge; $text =~ s{(\\image\s*)\[([^\]]+)\]} {"${1}[" . relative_url ($2, $metadata->{'index-base'}) . ']'}ge; print $file $text; print $file "\\class(footer)[$date \\entity[mdash]\n"; print $file " \\link[$entry->{link}]\n"; print $file " [Permanent link]]\n\n"; } if ($metadata->{'index-suffix'}) { print $file $metadata->{'index-suffix'}, "\n"; } print $file "\\signature\n"; } ############################################################################## # Main routine ############################################################################## $| = 1; my $fullpath = $0; $0 =~ s%.*/%%; # Parse command-line options. my ($base, $help, $version); Getopt::Long::config ('bundling'); GetOptions ('b|base=s' => \$base, 'h|help' => \$help, 'v|version' => \$version) or exit 1; if ($help) { print "Feeding myself to perldoc, please wait....\n"; exec ('perldoc', '-t', $fullpath); } elsif ($version) { my $version = join (' ', (split (' ', $ID))[1..3]); $version =~ s/,v\b//; $version =~ s/(\S+)$/($1)/; $version =~ tr%/%-%; print $version, "\n"; exit; } $base =~ s,/*$,/, if $base; my ($source) = @ARGV; die "Usage: $0 [-hv] [-b ] \n" unless $source; # Read in the changes. my (%metadata, @changes); parse_changes ($source, \%metadata, \@changes); # Now, the output key tells us what files to write out. my @output; if ($metadata{output}) { @output = split (' ', $metadata{output}); } else { @output = ('*:rss:index.rss'); } for my $output (@output) { my ($tags, $format, $file) = split (':', $output); my $path; if ($base && $file !~ m,^/,) { $path = "$base$file"; } else { $path = $file; } my $prettyfile = $path; $prettyfile = ".../$prettyfile" unless $prettyfile =~ m,^/,; next if (-f $path && -M $path <= -M $source); my @tags = split (',', $tags); my @interest; for my $change (@changes) { if ($tags eq '*' || intersect ($change->{tags}, \@tags)) { push (@interest, $change); } } if ($format eq 'thread') { print "Generating thread file $prettyfile\n"; open (THREAD, '>', $path) or die "$0: cannot create $path: $!\n"; thread_output (\*THREAD, \%metadata, \@interest); close THREAD; } elsif ($format eq 'rss') { if (@interest > $metadata{recent}) { splice (@interest, $metadata{recent}); } print "Generating RSS file $prettyfile\n"; open (RSS, '>', $path) or die "$0: cannot create $path: $!\n"; rss_output (\*RSS, $file, \%metadata, \@interest); close RSS; } elsif ($format eq 'index') { if (@interest > $metadata{recent}) { splice (@interest, $metadata{recent}); } print "Generating index file $prettyfile\n"; open (INDEX, '>', $path) or die "$0: cannot create $path: $!\n"; index_output (\*INDEX, \%metadata, \@interest); close INDEX; } } ############################################################################## # Documentation ############################################################################## =head1 NAME spin-rss - Generate RSS and thread from a feed description file =head1 SYNOPSIS B [B<-hv>] [B<-b> I] I =head1 REQUIREMENTS Perl 5.006 or later and the Date::Parse module, which is part of the TimeDate distribution on CPAN. B is required for anything other than simple entries with the text of the entry inline in the feed description file. =head1 DESCRIPTION B reads as input a feed description file consisting of simple key/value pairs and writes out either thread (for input to B) or RSS. The feed description consists of a leading block of metadata and then one block per entry in the feed. Each block can either include the content of the entry or can reference an external thread file, in several formats, for the content. The feed description file defines one or more output files in the Output field of the metadata. Output files are only regenerated if they are older than the input feed description file. B is designed for use with B. It relies on B to convert thread to HTML, both for inclusion in RSS feeds and for post-processing of generated thread files. B is invoked automatically by B when B encounters an F<.rss> file in a directory it is processing. =head1 OPTIONS =over 4 =item B<-b> I, B<--base>=I By default, B output files are relative to the current working directory. If the B<-b> option is given, output files will be relative to I instead. Output files specified as absolute paths will not be affected. B always passes this option to B. =item B<-h>, B<--help> Print out this documentation (which is done simply by feeding the script to C). =item B<-v>, B<--version> Print out the version of B and exit. =back =head1 INPUT LANGUAGE The input for B is normally a F<.rss> file in a tree being processed by B, but may be any file name passed to B. The file consists of one or more blocks of RFC-2822-style fields with values, each separated by a blank line. Each field and value looks like an e-mail header field, including possible continuation lines: Field: value continuation of value Any line beginning with whitespace is considered a continuation of the previous line. If a value should contain a blank line, indicate that blank line with a continuation line containing only a period. For example: Field: first paragraph . second paragraph =head2 Metadata The first block of the file sets the metadata for this set of output. The following fields are supported: =over 4 =item Base The base URL for entries in this file. All links in subsequent blocks of the file, if not absolute URLs, are treated as relative to this URL and are made absolute by prepending this URL. Always specify this key unless all Link fields in the remainder of the file use absolute URLs. This field value is also used as the element in the RSS feed, indicating the web site corresponding to this feed. =item Description The description of the feed, used only in the RSS output. This should always be set if there are any RSS output files. =item Index-Base The base URL for output files of type C. This is used to canonicalize relative URLs and should be the URL to the directory containing the HTML file that will result from processing the thread output. This should be set if there are any output files of type C; if it isn't set, relative links may be rewritten incorrectly. =item Index-Prefix When generating output files of type C, use the value as the initial content of the generated thread. This field should almost always be set if any output files of type C are defined. It will contain such things as the \heading command, any prologue material, initial headings, and so forth. =item Index-Suffix When generating output files of type C, append the value to the end of the generated thread. The \signature command is always appended and should not be included here. Set this field only if there is other thread that needs to be appended (such as closing brackets for \div commands). =item Language The language of the feed, used only in the RSS output. This should always be set if there are any RSS output files. Use C for US English. =item Output Specifies the output files for this input file in the form of a whitespace-separated list of output specifiers. This field must always be set. An output specifier is of the form I:I:I, where I is the output file (always a relative path), I is the type of output, and I indicates which entries to include in this file. I is a comma-separated list of tags or the special value C<*>, indicating all tags. There are three types of output: =over 4 =item index Output thread containing all recent entries. This output file honors the Recent field similar to RSS output and is used to generate something akin to a journal or blog front page: an HTML version of all recent entries. It only supports external entries (entries with Journal or Review fields). The Index-Base and Index-Prefix (and possibly Index-Suffix) fields should be set. For output for entries with simple descriptions included in the input file, see the C output type. =item rss Output an RSS file. B only understands the RSS 2.0 output format. The Description, Language, RSS-Base, and Title fields should be set to provide additional metadata for the output file. =item thread Output thread containing all entries in this input file. This should only be used for input files where all entries have their description text inline in the input file. Every entry will be included. The output will be divided into sections by month, and each entry will be in a description list, with the title prefixed by the date. The Thread-Prefix field should be set. For output that can handle entries from external files, see the C output type. =back =item Recent Sets the number of recent entries to include in output files of type C or C (but not C, which will include the full contents of the file). If this field is not present, the default is 15. =item RSS-Base The base URL for RSS files generated by this file. Each generated RSS file should have a link back to itself, and that link will be formed by taking the output name and prepending this field. =item Thread-Prefix When generating thread output from this file, use the value as the initial content of the generated thread. This field should almost always be set if any output files of type C are defined. It will contain such things as the \heading command, any prologue material, initial headings, and so forth. =item Title The title of the feed, used only in the RSS output. This should always be set if there are any RSS output files. =back =head2 Entries After the first block, each subsequent block in the input file defines an entry. Entries take the following fields: =over 4 =item Date The date of this entry in ISO date format (YYYY-MM-DD HH:MM). This field is required. =item Description The inline contents of this entry. One and only one of this field, Journal, or Review should be present. Description fields can only be used with output types of C or C. =item Journal Specifies that the content of this entry should be read from an external thread file given by the value of this field. The contents of that file are expected to be in the thread format used by my journal entries: specifically, everything is ignored up to the first \h1 and after \class(date), and the \class(date) line is stripped off (the date information from the Date field is used instead). One and only one of this field, Description, or Review should be present. =item Link The link to the page referenced by this entry. The link is relative to the Base field set in the input filel metadata. This field is required. =item Review Specifies that the content of this entry should be read from an external thread file given by the value ofo this field. The contents of that file are expected to be in the thread format used by my book reviews. Many transformations are applied to entries of this sort based on the format used by my book reviews and the URL layout they use, none of which is documented at present. For the time being, see the source code for what transformations are done. This support will require modification for use by anyone else. One and only one of this field, Description, or Review should be present. =item Tags Whitespace-separated tags for this entry, used to determine whether this entry will be included in a given output file given its output specification. In addition to any tags listed here, any entry with a Review field will automatically have the C tag. Entries may have no tags, in which case they're only included in output files with a tag specification of C<*>. =item Title The title of the entry. This field is required. =back =head1 BUGS The separate C and C output types are a historical artifact and should be merged in some way. This will likely also require a way of configuring different caps on number of included entries in different output specifications. The file format for review entries is not specified and involves a lot of ad hoc heuristics that work with my book review format. Support for additional RSS standards would be nice, although in practice everything seems to be able to cope with RSS 2.0. =head1 NOTES RSS 2.0 was chosen as the output format because it supports GUIDs for entries separate from the entry URLs and hence supports multiple entries for the same URL, something that I needed for an RSS feed of recent changes to my entire site. =head1 SEE ALSO spin(1) =head1 AUTHOR Russ Allbery =head1 COPYRIGHT AND LICENSE Copyright 2008 Russ Allbery . This program is free software; you may redistribute it and/or modify it under the same terms as Perl itself. =cut