/ 
Data-Downloader-0.9904
River stage zero No dependents
/ Data::Downloader

NAME

Data::Downloader -- Download and organize files using RSS feeds and templates

SYNOPSIS

#!/usr/bin/env perl

use Data::Downloader -init_logging => "DEBUG";
use strict;

# Get pictures of apples.  Store them using md5's, make symlinks using date and tags

my $images = Data::Downloader::Repository->new(
    name           => "images",
    storage_root   => "/usr/local/datastore/data",
    cache_strategy => "Keep",
    feeds          => [
        {
            name => "flickr",
            feed_template => 'http://api.flickr.com/services/feeds/photos_public.gne?tags=<tags>&lang=en-us&format=rss_200',
            file_source => {
                url_xpath      => 'media:content/@url',
                filename_xpath => 'media:content/@url',
                filename_regex => '/([^/]*)$',
            },
            metadata_sources => [
                { name => 'date_taken', xpath => 'dc:date.Taken' },
                { name => 'tags',       xpath => 'media:category' },
            ],
        },
    ],
    metadata_transformations => [
        {
            input         => "tags",
            output        => "tag",
            function_name => "split",
        },
    ],
    linktrees => [
        {
            root          => "/usr/local/datastore/by_tag",
            condition     => undef,
            path_template => "<tag>"
        },
        {
            root          => "/usr/local/datastore/by_date",
            condition     => undef,
            path_template => "<date_taken:%Y/%m/%d>"
        },
    ],
);

$images->load(speculative => 1) or $images->save;

for my $feed ($images->feeds) {
    $feed->refresh(tags => "apples");
}

$images->download_all;

DESCRIPTION

Data::Downloader allows one to maintain local repositories of files and file metadata based on RSS feeds which describe the files. It stores files uniquely using MD5 sums, but allows for the creation and maintenance of trees of symbolic links to the downloaded files.

An SQLite database is used to store both the configuration and information about the files that have been downloaded. This file is automatically created in $HOME/.data_downloader.db (by default).

This distribution also conatins a command-line tool, dado, for managing the repository.

CONFIGURATION

See Data::Downloader::Config.

UPDATING RSS FEEDS

Read about this in Data::Downloader::Feed.

DOWNLOADING FILES

Take a look at Data::Downloader::File.

Check out Data::Downloader::Linktree.

SEE ALSO

dado

Data::Downloader::Config

Data::Downloader::Repository

Data::Downloader::Feed

Data::Downloader::DB

Data::Downloader::Cache

Data::Downloader::Linktree

Rose

SCHEMA

__DATA__

-- See also Rose::DB::Object::Loader

/*
 * Every table below corresponds to a class named Data::Downloader::<TableName>.
 * Every class has accessors and mutators whose names are the same as the column names.
 * Foreign keys also beget methods in both the parent and child classes, e.g.
 * e.g. $repository->files and $file->repository_obj.
 */

  /*** static tables : populated once during configuration ***/

--- A repository has a cache strategy.

create table repository (
    id                integer primary key,
    name              varchar(255) not null unique,
    storage_root      text not null,
    file_url_template text,
    cache_strategy    text not null, -- e.g. "LRU" corresponds to Data::Downloader::Cache::LRU
    cache_max_size    integer -- in bytes
);

--- A repository has many feeds.

create table feed (
    id                integer primary key,
    repository        integer not null references repository(id),
    name              varchar(255) not null unique,
    feed_template     text
);

--- A repository has many disks.

create table disk (
    id                integer primary key,
    repository        integer not null references repository(id),
    root              varchar(255) not null unique
);

--- A feed has many default parameters

create table feed_parameter (
   id    integer primary key,
   feed  integer not null references feed(id),
   name  text not null,
   default_value text
);

--- A feed has a file source.

create table file_source (
    feed             integer primary key not null references feed(id),
    url_xpath        text,
    urn_xpath        text,  -- unique identifier for files
    md5_xpath        text,
    filename_xpath   text,
    filename_regex   text   -- apply to whatever is extracted from the xpath
);

--- A feed has many metadata sources.

create table metadata_source (
    id               integer primary key,
    feed             integer references feed(id),
    name             text not null unique,
    xpath            text
);

--- A repository has many linktrees.

create table linktree (
    id               integer primary key,
    repository       integer references repository(id),
    root             text,
    condition        text,      -- SQL::Abstract clause
    path_template    text,      -- String::Template string
    unique(root)
);

--- A repository has many metadata_transformationss.

 create table metadata_transformation (
     id               integer primary key,
     input            text, -- references metadata_source.name or this table
     output           text not null,
     repository       integer references repository(id),
     function_name    text, -- function to apply
     function_params  text,
     order_key        integer not null default 1,
     unique(order_key)
 );

/*** dynamic tables  : populated when files are downloaded, rss feeds are updated, etc. ***/

 create table stat_info (
     id integer primary key,
     repository integer not null references repository(id),
     last_stat_update datetime,
     last_fsck datetime,
     unique(repository)
 );

--- metadata

create table metadatum (
    id     integer primary key,
    file   integer references file(id),
    name   varchar(255) references metadata_source(name),
    value  text,
    unique (file,name)
);

--- A repository has many files.

create table file (
   id                 integer primary key,
   repository         integer not null references repository(id),
   filename           text not null,
   md5                char(32),
   url                text,
   urn                text,
   size               integer,
   on_disk            integer, -- boolean
   disk               integer references disk(id),
   atime              datetime, -- stat(file)->atime
   unique(md5),
   unique(filename),
   unique(urn)
);

--- A file has many log entries --- (if $ENV{DATA_DOWNLOADER_GATHER_STATS} is set).

create table log_entry (
   id integer primary key,
   file integer not null references file(id),
   requested_at datetime not null,
   cache_hit integer, -- boolean
   completed_at datetime,
   prog text, -- $0
   pid integer, -- $$
   uid text, -- $<
   note text -- $ENV{DATA_DOWNLOADER_LOG_NOTE}, e.g. app info
);

--- A file has many expirations

create table file_removal (
   id integer primary key,
   file integer not null references file(id),
   expired_on datetime,
   algorithm, -- the expiration algorithm
   prog text, -- $0
   pid integer, -- $$
   uid text, -- $<
   note text -- $ENV{DATA_DOWNLOADER_FILE_REMOVAL_NOTE},
);

--- A file has many symlinks.

create table symlink (
   id                 integer primary key,
   linkname           text not null, -- linktree.path_template + file's metadata
   file               integer not null references file(id),
   linktree           integer not null references linktree(id),
   unique(linkname)
);

/***\

AUTHOR

Brian Duggan

Phillip Durbin

Stuart Pineo

Arnold Martin

Graham Ollis

Curt Tilmes

Michael Walters