mirror of https://github.com/vbatts/sl-feeds.git
Browse Source
and ability to export to feeds Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>pull/4/head
GPG Key ID:
10937E57733F1362
14 changed files with 1750 additions and 535 deletions
@ -0,0 +1,28 @@
|
||||
Copyright (c) 2016 Vincent Batts, Raleigh, NC, USA |
||||
|
||||
All rights reserved. |
||||
|
||||
Redistribution and use in source and binary forms, with or without |
||||
modification, are permitted provided that the following conditions are met: |
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this |
||||
list of conditions and the following disclaimer. |
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice, |
||||
this list of conditions and the following disclaimer in the documentation |
||||
and/or other materials provided with the distribution. |
||||
|
||||
3. Neither the name of the copyright holder nor the names of its contributors |
||||
may be used to endorse or promote products derived from this software without |
||||
specific prior written permission. |
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND |
||||
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
||||
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE |
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE |
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL |
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR |
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER |
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, |
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE |
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||||
@ -1,15 +0,0 @@
|
||||
hack scripts |
||||
|
||||
They accomplish going from ChangeLog.txt -> RSS feeds that folks can subscribed to. |
||||
Ultimately ending up at http://www.slackware.com/~vbatts/feeds/ |
||||
|
||||
These are a mess, but still work. |
||||
|
||||
I wrote a utility called slack-utils (https://github.com/vbatts/slack-utils/) |
||||
with a ruby gem (https://rubygems.org/gems/slack-utils). The python script |
||||
(`./bin/changelog_http_poll.py`) is called by a crontab. Python walks the |
||||
changelog.txt, then fetches from the http mirror the modified time. If it http |
||||
has the new version, then run the ruby script (`./bin/gen_changlog_rss.rb`). |
||||
This is what parses the changelog and returns RSS. Which is then written to the |
||||
corresponding file for public consumption. |
||||
|
||||
@ -0,0 +1,18 @@
|
||||
# sl-feeds |
||||
|
||||
This is for proccessing Slackware ChangeLog.txt -> RSS feeds that folks can |
||||
subscribed to. |
||||
|
||||
Ultimately ending up at http://www.slackware.com/~vbatts/feeds/ |
||||
|
||||
## Usage |
||||
|
||||
```bash |
||||
go get github.com/vbatts/sl-feeds |
||||
``` |
||||
|
||||
crontab like: |
||||
|
||||
``` |
||||
0 */2 * * * ~/bin/sl-feeds -q || echo "$(date): failed to poll changelogs" | mail -s "[slackrss] changelog_http_poll failed $(date +%D)" me@example.com |
||||
``` |
||||
@ -1,62 +0,0 @@
|
||||
#!/home/vbatts/opt/bin/ruby |
||||
|
||||
#require 'fileutils' |
||||
require 'logger' |
||||
require 'tempfile' |
||||
require 'stringio' |
||||
|
||||
require 'rubygems' |
||||
require 'slackware' |
||||
require 'slackware/changelog/rss' |
||||
|
||||
#include FileUtils |
||||
|
||||
$LOG = Logger.new(STDERR) |
||||
$LOG.level = Logger::WARN |
||||
|
||||
FEEDS_BASE_DIR = "/home/vbatts/public_html/feeds/" |
||||
#url = 'http://alphageek.dyndns.org/linux/slackware-packages/slack-13.1/ChangeLog.txt' |
||||
# Sun Feb 13 08:44:35 PST 2011 |
||||
# new url |
||||
URL = 'http://alphageek.dyndns.org/mirrors/alphageek/slackware-%s/ChangeLog.txt' |
||||
|
||||
VERSIONS = %w{ 14.0 14.1 } |
||||
|
||||
def url(ver) |
||||
URL % ver |
||||
end |
||||
|
||||
if ARGV.include?('-v') |
||||
$LOG.level = Logger::DEBUG |
||||
end |
||||
|
||||
VERSIONS.each {|ver| |
||||
begin |
||||
#tmp_file = File.open("/tmp/vbatts/alpha_log-#{(rand*1000).to_i}.xxx", "w+") |
||||
tmp_file = Tempfile.new("alpha_log") |
||||
$LOG.debug('tmp_file') { tmp_file } |
||||
|
||||
strio = StringIO.new() |
||||
$LOG.debug('created ') { strio } |
||||
|
||||
buffer = `lynx -source #{url(ver)}` |
||||
$LOG.debug('buffer length') { buffer.length } |
||||
|
||||
tmp_file.write(buffer) |
||||
tmp_file.flush |
||||
|
||||
changelog = Slackware::ChangeLog.new(tmp_file.path) |
||||
changelog.parse |
||||
strio.write(changelog.to_rss( |
||||
:noimage => true, |
||||
:title => "alphageek's #{ver} ChangeLog", |
||||
:url => url(ver))) |
||||
ensure |
||||
strio.seek(0) |
||||
tmp_file.close |
||||
end |
||||
feed_file = File.open(FEEDS_BASE_DIR + "alphageek-#{ver}_ChangeLog.rss", "w+") |
||||
$LOG.debug('feed_file') { feed_file } |
||||
feed_file.write(strio.read()) |
||||
feed_file.close |
||||
} |
||||
@ -1,194 +0,0 @@
|
||||
#!/usr/bin/env python |
||||
# Mon Oct 17 08:25:29 PDT 2011 |
||||
# copyright 2011 Vincent Batts, Vienna, VA, USA |
||||
|
||||
# switching from an inotify watcher, to an http poll |
||||
# since what lands on connie.slackware.com usually doesn't go public |
||||
# immediately |
||||
|
||||
|
||||
import os |
||||
import sys |
||||
import glob |
||||
import time |
||||
from datetime import datetime |
||||
from datetime import timedelta |
||||
from time import mktime |
||||
import urllib2 |
||||
import anydbm |
||||
|
||||
DEFAULT_DB = os.path.join(os.getenv('HOME'), '.slackware_changelog.db') |
||||
DEFAULT_URL = "http://slackware.osuosl.org/" |
||||
SLACKWARE_DIR_PATH = "/mirrors/ftp.slackware.com/pub/slackware" |
||||
RSS_DIR_PATH = "/home/vbatts/public_html/feeds" |
||||
|
||||
''' |
||||
slackware-12.2_ChangeLog.rss |
||||
/home/vbatts/public_html/feeds/slackware-10.1_patches_ChangeLog.rss |
||||
/home/vbatts/public_html/feeds/slackware-8.1_patches_ChangeLog.rss |
||||
>>> for i in c.slackware_versions(): print i |
||||
... |
||||
/mirrors/ftp.slackware.com/pub/slackware/slackware64-13.0/ChangeLog.txt |
||||
/mirrors/ftp.slackware.com/pub/slackware/slackware-8.1/ChangeLog.txt |
||||
/mirrors/ftp.slackware.com/pub/slackware/slackware64-13.37/ChangeLog.txt |
||||
/mirrors/ftp.slackware.com/pub/slackware/slackware-13.0/ChangeLog.txt |
||||
/mirrors/ftp.slackware.com/pub/sla |
||||
''' |
||||
|
||||
def rss_files(): |
||||
for item in glob.glob(RSS_DIR_PATH + "/*.rss"): |
||||
yield item |
||||
|
||||
def rss_files_format(str): |
||||
if str.startswith(RSS_DIR_PATH + "/"): |
||||
str = str[len(RSS_DIR_PATH + "/"):] |
||||
if str.endswith(".rss"): |
||||
str = str[:-4] |
||||
str = str + '.txt' |
||||
return str.replace('_','/') |
||||
|
||||
def rss_files_cleaned(): |
||||
for i in rss_files(): |
||||
yield rss_files_format(i) |
||||
|
||||
def slackware_versions(): |
||||
changes = glob.glob(SLACKWARE_DIR_PATH + "/*/ChangeLog.txt") |
||||
patches = glob.glob(SLACKWARE_DIR_PATH + "/*/patches/ChangeLog.txt") |
||||
for item in changes + patches: |
||||
yield item |
||||
|
||||
def slackware_versions_format(str): |
||||
if str.startswith(SLACKWARE_DIR_PATH + "/"): |
||||
str = str[len(SLACKWARE_DIR_PATH + "/"):] |
||||
if str.endswith("/"): |
||||
str = str[:-1] |
||||
if str.startswith("/"): |
||||
str = str[1:] |
||||
if str.endswith(".txt"): |
||||
str = str[:-4] |
||||
return str.replace('/','_') |
||||
|
||||
def slackware_versions_strip(): |
||||
for i in slackware_versions(): |
||||
yield i[len(SLACKWARE_DIR_PATH + "/"):] |
||||
|
||||
def slackware_versions_rss(): |
||||
for i in slackware_versions(): |
||||
yield slackware_versions_format(i) |
||||
|
||||
def process_changelog_rss(pathname): |
||||
if os.path.basename(pathname) == "ChangeLog.txt": |
||||
print "%f: proccessing %s" % (time.time(), pathname) |
||||
# XXX REPLACE ME!! |
||||
cmd = "/home/vbatts/opt/bin/ruby /home/vbatts/bin/gen_changlog_rss.rb %s" % pathname |
||||
print cmd |
||||
print os.system(cmd) |
||||
else: |
||||
print '[WARN] "%s" is not a ChangeLog.txt file' % pathname |
||||
|
||||
def db_setup(name = DEFAULT_DB): |
||||
try: |
||||
return anydbm.open(name, 'c') |
||||
except: |
||||
return None |
||||
|
||||
def db_teardown(db): |
||||
try: |
||||
return db.close() |
||||
except: |
||||
return None |
||||
|
||||
def db_add_ts(db, key, val): |
||||
if type(val) == float: |
||||
db[key] = str(val) |
||||
if type(val) == datetime: |
||||
db[key] = str(unix_time(val)) |
||||
return db[key] |
||||
|
||||
def db_get_ts(db, key): |
||||
try: |
||||
return datetime.fromtimestamp(float(db[key])) |
||||
except KeyError: |
||||
return None |
||||
|
||||
def unix_time(dt): |
||||
return mktime(dt.timetuple())+1e-6*dt.microsecond |
||||
|
||||
def time_from_header(str): |
||||
return datetime.strptime(str, "%a, %d %b %Y %H:%M:%S %Z") |
||||
|
||||
def get_remote_header(url, header): |
||||
try: |
||||
req = urllib2.Request(url) |
||||
resp = urllib2.urlopen(req) |
||||
return resp.headers.getheader(header) |
||||
except: |
||||
return None |
||||
|
||||
def get_remote_time_str(url): |
||||
return get_remote_header(url,"last-modified") |
||||
|
||||
def get_remote_time(url): |
||||
time_str = get_remote_time_str(url) |
||||
if time_str: |
||||
return time_from_header(time_str) |
||||
else: |
||||
return None |
||||
|
||||
def get_local_time(path): |
||||
try: |
||||
time_flt = os.stat(path).st_mtime |
||||
return datetime.fromtimestamp(time_flt) |
||||
except: |
||||
return None |
||||
|
||||
def main(args): |
||||
try: |
||||
db = db_setup() |
||||
if db == None: |
||||
print "ERROR: could not setup database at %s" % DEFAULT_DB |
||||
return 1 |
||||
|
||||
for i in slackware_versions_strip(): |
||||
# i'm not going to worry about this file, right now |
||||
if i == 'slackware/ChangeLog.txt': |
||||
continue |
||||
|
||||
rss_file_name = os.path.join(RSS_DIR_PATH, |
||||
slackware_versions_format(i) + ".rss") |
||||
rss_ts = get_local_time(rss_file_name) |
||||
curr_ts = get_local_time(os.path.join(SLACKWARE_DIR_PATH, i)) |
||||
prev_ts = db_get_ts( db, "local_" + i) |
||||
|
||||
# Go no further for this file |
||||
if curr_ts == prev_ts and os.path.exists(rss_file_name) and rss_ts > prev_ts: |
||||
print '[INFO] Local time of "%s" is same as the database has' % i |
||||
continue |
||||
|
||||
db_add_ts( db, "local_" + i, curr_ts) |
||||
|
||||
remote_ts = get_remote_time(DEFAULT_URL + i) |
||||
print '[INFO] inserting remote_%s: %s' % (i,remote_ts) |
||||
db_add_ts( db, "remote_" + i, remote_ts) |
||||
|
||||
if prev_ts == None or (remote_ts - prev_ts) == timedelta(hours=7): |
||||
print '[INFO] local and remote ChangeLog times match' |
||||
if rss_ts == None: |
||||
print '[INFO] RSS file (%s) does not exist' % (rss_ts) |
||||
print '[INFO] Processing "%s"' % rss_file_name |
||||
process_changelog_rss(os.path.join(SLACKWARE_DIR_PATH, i)) |
||||
elif prev_ts == None or rss_ts < prev_ts: |
||||
print '[INFO] RSS file (%s) is older than the ChangeLog (%s)' % (rss_ts, prev_ts) |
||||
print '[INFO] Processing "%s"' % rss_file_name |
||||
process_changelog_rss(os.path.join(SLACKWARE_DIR_PATH, i)) |
||||
else: |
||||
print '[INFO] RSS seems current' |
||||
finally: |
||||
try: |
||||
os.wait() |
||||
except: |
||||
pass |
||||
db_teardown(db) |
||||
|
||||
if __name__ == "__main__": sys.exit(main(sys.argv[1:])) |
||||
|
||||
@ -1,39 +0,0 @@
|
||||
#!/usr/bin/env python |
||||
|
||||
import os |
||||
import sys |
||||
import glob |
||||
import time |
||||
|
||||
sys.path.insert(0, "/home/vbatts/opt/lib/python2.5/site-packages") |
||||
sys.path.insert(0, "/home/vbatts/opt/lib/python2.5") |
||||
import pyinotify |
||||
|
||||
dir_path = "/mirrors/ftp.slackware.com/pub/slackware" |
||||
|
||||
def process_changelog_rss(event): |
||||
if os.path.basename(event.pathname) == "ChangeLog.txt": |
||||
print "%f: proccessing %s" % (time.time(), event) |
||||
os.system("/home/vbatts/opt/bin/ruby /home/vbatts/bin/gen_changlog_rss.rb %s" % event.pathname) |
||||
|
||||
def main(args): |
||||
wm = pyinotify.WatchManager() |
||||
|
||||
notifier = pyinotify.Notifier(wm) |
||||
|
||||
for dir in glob.glob(dir_path + "/*/"): |
||||
if os.path.exists(dir + "ChangeLog.txt"): |
||||
print "%f: Adding watch for %s" % (time.time(), dir) |
||||
wm.add_watch(dir, pyinotify.IN_MOVED_TO, rec=False, proc_fun=process_changelog_rss) |
||||
|
||||
for dir in glob.glob(dir_path + "/*/patches/"): |
||||
print "%f: Adding watch for %s" % (time.time(), dir) |
||||
wm.add_watch(dir, pyinotify.IN_MOVED_TO, rec=False, proc_fun=process_changelog_rss) |
||||
|
||||
#wm.add_watch("/home/vbatts/", pyinotify.IN_MOVED_TO, rec=False, proc_fun=process_changelog_rss) |
||||
|
||||
notifier.loop() |
||||
|
||||
|
||||
if __name__ == "__main__": main(sys.argv[1:]) |
||||
|
||||
@ -1,155 +0,0 @@
|
||||
#!/usr/bin/env ruby |
||||
# Sun Jan 23 11:30:53 PST 2011 |
||||
# Created by vbatts, vbatts@hashbangbash.com |
||||
|
||||
$PROGRAM_NAME = File.basename(__FILE__) |
||||
|
||||
require 'find' |
||||
|
||||
require 'rubygems' |
||||
require 'ruby-prof' |
||||
require 'slackware' |
||||
require 'slackware/changelog/rss' |
||||
require 'rb-inotify' |
||||
|
||||
|
||||
BASE_URL = "http://slackware.osuosl.org/" |
||||
MIRROR_BASE_DIR = "/mirrors/ftp.slackware.com/pub/slackware/" |
||||
FEEDS_BASE_DIR = "/home/vbatts/public_html/feeds/" |
||||
RE_REPO_NAME = Regexp.new(/slackware(\d{2})?-(\d+\.\d+|current)\/(patches)?\/?.*/) |
||||
|
||||
def generate_new_if_none |
||||
files = [] |
||||
|
||||
Find.find(MIRROR_BASE_DIR) {|file| |
||||
relative_name = file.sub(MIRROR_BASE_DIR, "") |
||||
if File.basename(file) == "ChangeLog.txt" |
||||
if not(relative_name.include?("zipslack")) |
||||
files << relative_name |
||||
Find.prune |
||||
end |
||||
end |
||||
# putting this check *after* the one above, |
||||
# lets us get the patches directories too |
||||
# while still getting a bit of speed (1.5s) |
||||
if relative_name.split("/").count > 2 |
||||
Find.prune |
||||
end |
||||
} |
||||
puts "%f: watching %d changelogs" % [Time.now.to_f, files.count] |
||||
files.each {|file| |
||||
m = RE_REPO_NAME.match file |
||||
if m[3].nil? |
||||
file_name = "%sslackware%s-%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2]] |
||||
else |
||||
file_name = "%sslackware%s-%s_%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2], m[3]] |
||||
end |
||||
unless File.exist?(file_name) |
||||
c_file = MIRROR_BASE_DIR + file |
||||
changelog = Slackware::ChangeLog.new(c_file, :version => m[2]) |
||||
changelog.opts[:arch] = m[1] unless m[1].nil? |
||||
if m[3].nil? |
||||
changelog.opts[:url] = "%sslackware%s-%s/ChangeLog.txt" % [BASE_URL, m[1], m[2]] |
||||
feed = File.open( "%sslackware%s-%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2]], "w+") |
||||
else |
||||
changelog.opts[:url] = "%sslackware%s-%s/%s/ChangeLog.txt" % [BASE_URL, m[1], m[2], m[3]] |
||||
feed = File.open( "%sslackware%s-%s_%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2], m[3]], "w+") |
||||
end |
||||
changelog.parse |
||||
puts "%f: Making a first feed: %s" % [Time.now.to_f, feed.path] |
||||
feed << changelog.to_rss |
||||
feed.close |
||||
changelog = nil |
||||
end |
||||
} |
||||
end |
||||
|
||||
def run_notifier |
||||
n = INotify::Notifier.new |
||||
dirs = Dir.glob(MIRROR_BASE_DIR + "*") |
||||
dirs.concat(Dir.glob(MIRROR_BASE_DIR + "*/patches/")) |
||||
dirs.each {|dir| |
||||
next unless File.exist?(File.join(dir, "ChangeLog.txt")) |
||||
puts "%f: working with %s" % [Time.now.to_f, dir] |
||||
n.watch(dir, :moved_to) {|mfile| |
||||
file_name = mfile.absolute_name |
||||
if File.basename(file_name) == "ChangeLog.txt" |
||||
puts "%f: looking into %s" % [Time.now.to_f, file_name] |
||||
match_data = RE_REPO_NAME.match(file_name) |
||||
|
||||
unless match_data.nil? |
||||
changelog = Slackware::ChangeLog.new(file_name, :version => match_data[2]) |
||||
changelog.opts[:arch] = match_data[1] unless match_data[1].nil? |
||||
|
||||
if match_data[3].nil? |
||||
changelog.opts[:url] = "%sslackware%s-%s/ChangeLog.txt" % [ |
||||
BASE_URL, |
||||
match_data[1], |
||||
match_data[2] |
||||
] |
||||
feed = File.open( "%sslackware%s-%s_ChangeLog.rss" % [ |
||||
FEEDS_BASE_DIR, |
||||
match_data[1], |
||||
match_data[2] |
||||
], "w+") |
||||
else |
||||
changelog.opts[:url] = "%sslackware%s-%s/%s/ChangeLog.txt" % [ |
||||
BASE_URL, |
||||
match_data[1], |
||||
match_data[2], |
||||
match_data[3] |
||||
] |
||||
feed = File.open( "%sslackware%s-%s_%s_ChangeLog.rss" % [ |
||||
FEEDS_BASE_DIR, |
||||
match_data[1], |
||||
match_data[2], |
||||
match_data[3] |
||||
], "w+") |
||||
end |
||||
begin |
||||
changelog.parse |
||||
rescue StandardError => ex |
||||
puts "%f: %s" % [Time.now.to_f, ex.message] |
||||
puts "%f: %s" % [Time.now.to_f, file_name] |
||||
next |
||||
end |
||||
|
||||
puts "%f: parsed %s to %s" % [Time.now.to_f, file_name, feed.path] |
||||
|
||||
feed << changelog.to_rss |
||||
feed.close |
||||
changelog = nil |
||||
end |
||||
end |
||||
} |
||||
} |
||||
begin |
||||
n.run |
||||
rescue Interrupt |
||||
end |
||||
end |
||||
|
||||
## Main |
||||
|
||||
#generate_new_if_none() |
||||
begin |
||||
RubyProf.start |
||||
run_notifier() |
||||
ensure |
||||
result = RubyProf.stop |
||||
|
||||
RubyProf.measure_mode = RubyProf::PROCESS_TIME |
||||
RubyProf.measure_mode = RubyProf::WALL_TIME |
||||
RubyProf.measure_mode = RubyProf::CPU_TIME |
||||
#RubyProf.measure_mode = RubyProf::ALLOCATIONS |
||||
#RubyProf.measure_mode = RubyProf::MEMORY |
||||
#RubyProf.measure_mode = RubyProf::GC_RUNS |
||||
#RubyProf.measure_mode = RubyProf::GC_TIME |
||||
|
||||
output_file_name = File.join(ENV["HOME"],"%s-%s%s" % [Time.now.to_i.to_s,File.basename(__FILE__),".log"]) |
||||
output_file = File.open(output_file_name, "w+") |
||||
printer = RubyProf::FlatPrinter.new(result) |
||||
printer.print(output_file,0) |
||||
puts "%f: %s written" % [Time.now.to_f, output_file_name] |
||||
output_file.close |
||||
end |
||||
@ -1,68 +0,0 @@
|
||||
#!/home/vbatts/opt/bin/ruby |
||||
|
||||
require 'logger' |
||||
|
||||
$log = Logger.new(STDERR) |
||||
$log.level = Logger::DEBUG |
||||
|
||||
# put this in a loader function, because the |
||||
# rss library is SOO SLOW to load. we don't want to load it, |
||||
# if the script is going to fail early. |
||||
def load_libs() |
||||
require 'rubygems' |
||||
require 'slackware' |
||||
require 'slackware/changelog/rss' |
||||
require 'rb-inotify' |
||||
end |
||||
|
||||
|
||||
BASE_URL = "http://slackware.osuosl.org/" |
||||
FEEDS_BASE_DIR = "/home/vbatts/public_html/feeds/" |
||||
RE_REPO_NAME = Regexp.new(/slackware(\d{2})?-(\d+\.\d+|current)\/(patches)?\/?.*/) |
||||
|
||||
def gen_file(file) |
||||
m = RE_REPO_NAME.match file |
||||
if m[3].nil? |
||||
file_name = "%sslackware%s-%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2]] |
||||
else |
||||
file_name = "%sslackware%s-%s_%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2], m[3]] |
||||
end |
||||
|
||||
if File.exist?(file_name) |
||||
if File.mtime(file) < File.mtime(file_name) |
||||
printf("%f: INFO: %s is newer than %s\n", Time.now, file, file_name) |
||||
end |
||||
end |
||||
|
||||
changelog = Slackware::ChangeLog.new(file) #, :version => m[2]) |
||||
opts = Hash.new |
||||
opts[:arch] = m[1] unless m[1].nil? |
||||
if m[3].nil? |
||||
opts[:url] = "%sslackware%s-%s/ChangeLog.txt" % [BASE_URL, m[1], m[2]] |
||||
feed = File.open( "%sslackware%s-%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2]], "w+") |
||||
else |
||||
opts[:url] = "%sslackware%s-%s/%s/ChangeLog.txt" % [BASE_URL, m[1], m[2], m[3]] |
||||
feed = File.open( "%sslackware%s-%s_%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2], m[3]], "w+") |
||||
end |
||||
changelog.parse |
||||
printf("%f: INFO: generating feed: %s\n", Time.now.to_f, feed.path) |
||||
feed << changelog.to_rss(opts) |
||||
feed.close |
||||
changelog = nil |
||||
end |
||||
|
||||
if ARGV.count == 0 |
||||
$log.error("#{Time.now}: ERROR: ChangeLog.txt files must be passed\n") |
||||
exit(2) |
||||
else |
||||
load_libs() |
||||
for file in ARGV |
||||
if File.exist?(file) |
||||
gen_file(file) |
||||
else |
||||
$log.warn("#{Time.now}: WARN: #{file} does not exist\n") |
||||
end |
||||
end |
||||
end |
||||
|
||||
# vim: set sts=2 sw=2 et ai: |
||||
@ -0,0 +1,51 @@
|
||||
package changelog |
||||
|
||||
import ( |
||||
"fmt" |
||||
"time" |
||||
|
||||
"github.com/gorilla/feeds" |
||||
) |
||||
|
||||
// ToFeed produces a github.com/gorilla/feeds.Feed that can be written to Atom or Rss
|
||||
func ToFeed(link string, entries []Entry) (*feeds.Feed, error) { |
||||
var newestEntryTime time.Time |
||||
var oldestEntryTime time.Time |
||||
|
||||
for _, e := range entries { |
||||
if e.Date.After(newestEntryTime) { |
||||
newestEntryTime = e.Date |
||||
} |
||||
if e.Date.Before(oldestEntryTime) { |
||||
oldestEntryTime = e.Date |
||||
} |
||||
} |
||||
|
||||
feed := &feeds.Feed{ |
||||
Title: "", |
||||
Link: &feeds.Link{Href: link}, |
||||
Description: "Generated ChangeLog.txt feeds by sl-feeds (github.com/vbatts/sl-feeds)", |
||||
Created: oldestEntryTime, |
||||
Updated: newestEntryTime, |
||||
} |
||||
feed.Items = make([]*feeds.Item, len(entries)) |
||||
for i, e := range entries { |
||||
feed.Items[i] = &feeds.Item{ |
||||
Created: e.Date, |
||||
Link: &feeds.Link{Href: ""}, |
||||
Description: e.ToChangeLog(), |
||||
} |
||||
|
||||
updateWord := "updates" |
||||
if len(e.Updates) == 1 { |
||||
updateWord = "update" |
||||
} |
||||
if e.SecurityFix() { |
||||
feed.Items[i].Title = fmt.Sprintf("%d %s. Including a %s!", len(e.Updates), updateWord, securityFixStr) |
||||
} else { |
||||
feed.Items[i].Title = fmt.Sprintf("%d %s.", len(e.Updates), updateWord) |
||||
} |
||||
} |
||||
|
||||
return feed, nil |
||||
} |
||||
@ -0,0 +1,38 @@
|
||||
package changelog |
||||
|
||||
import ( |
||||
"io/ioutil" |
||||
"os" |
||||
"testing" |
||||
) |
||||
|
||||
func TestFeed(t *testing.T) { |
||||
fh, err := os.Open("testdata/ChangeLog.txt") |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
defer fh.Close() |
||||
|
||||
e, err := Parse(fh) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
f, err := ToFeed("http://slackware.osuosl.org/slackware64-current/ChangeLog.txt", e) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
rss, err := f.ToRss() |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
//println(rss)
|
||||
if len(rss) == 0 { |
||||
t.Error("rss output is empty") |
||||
} |
||||
|
||||
if err := f.WriteRss(ioutil.Discard); err != nil { |
||||
t.Error(err) |
||||
} |
||||
} |
||||
@ -0,0 +1,124 @@
|
||||
package changelog |
||||
|
||||
import ( |
||||
"bufio" |
||||
"fmt" |
||||
"io" |
||||
"regexp" |
||||
"strings" |
||||
"time" |
||||
) |
||||
|
||||
const ( |
||||
dividerStr = `+--------------------------+` |
||||
securityFixStr = `(* Security fix *)` |
||||
dayPat = `^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s.*\d{4}$` |
||||
updatePat = `^([a-z].*/.*): (Added|Rebuilt|Removed|Updated|Upgraded)\.$` |
||||
) |
||||
|
||||
var ( |
||||
dayReg = regexp.MustCompile(dayPat) |
||||
updateReg = regexp.MustCompile(updatePat) |
||||
) |
||||
|
||||
// Parse takes in a slackware ChangeLog.txt and returns its collections of Entries
|
||||
func Parse(r io.Reader) ([]Entry, error) { |
||||
buf := bufio.NewReader(r) |
||||
entries := []Entry{} |
||||
curEntry := Entry{} |
||||
var curUpdate *Update |
||||
for { |
||||
line, err := buf.ReadString('\n') |
||||
if err != nil && err != io.EOF { |
||||
return nil, err |
||||
} |
||||
isEOF := err == io.EOF |
||||
trimmedline := strings.TrimSuffix(line, "\n") |
||||
|
||||
if trimmedline == dividerStr { |
||||
if curUpdate != nil { |
||||
curEntry.Updates = append(curEntry.Updates, *curUpdate) |
||||
curUpdate = nil |
||||
} |
||||
entries = append(entries, curEntry) |
||||
if isEOF { |
||||
break |
||||
} |
||||
curEntry = Entry{} |
||||
} else if dayReg.MatchString(trimmedline) { |
||||
// this date means it is the beginning of an entry
|
||||
t, err := time.Parse(time.UnixDate, trimmedline) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
curEntry.Date = t |
||||
} else if updateReg.MatchString(trimmedline) { |
||||
// match on whether this is an update line
|
||||
if curUpdate != nil { |
||||
curEntry.Updates = append(curEntry.Updates, *curUpdate) |
||||
curUpdate = nil |
||||
} |
||||
m := updateReg.FindStringSubmatch(trimmedline) |
||||
curUpdate = &Update{ |
||||
Name: m[1], |
||||
Action: m[2], |
||||
} |
||||
} else if curUpdate != nil && strings.HasPrefix(trimmedline, " ") { |
||||
curUpdate.Comment = curUpdate.Comment + line |
||||
} else { |
||||
// Everything else is a comment on the Entry
|
||||
curEntry.Comment = curEntry.Comment + line |
||||
} |
||||
|
||||
if isEOF { |
||||
break |
||||
} |
||||
} |
||||
return entries, nil |
||||
} |
||||
|
||||
// Entry is an section of updates (or release comments) in a ChangeLog.txt
|
||||
type Entry struct { |
||||
Date time.Time |
||||
Comment string |
||||
Updates []Update |
||||
} |
||||
|
||||
// SecurityFix is whether an update in this ChangeLog Entry includes a SecurityFix
|
||||
func (e Entry) SecurityFix() bool { |
||||
for _, u := range e.Updates { |
||||
if u.SecurityFix() { |
||||
return true |
||||
} |
||||
} |
||||
return false |
||||
} |
||||
|
||||
// ToChangeLog reformats the struct as the text for ChangeLog.txt output
|
||||
func (e Entry) ToChangeLog() string { |
||||
str := e.Date.Format(time.UnixDate) + "\n" |
||||
if strings.Trim(e.Comment, " \n") != "" { |
||||
str = str + e.Comment |
||||
} |
||||
for _, u := range e.Updates { |
||||
str = str + u.ToChangeLog() |
||||
} |
||||
return str |
||||
} |
||||
|
||||
// Update is a package or component that is updated in a ChangeLog Entry
|
||||
type Update struct { |
||||
Name string |
||||
Action string |
||||
Comment string |
||||
} |
||||
|
||||
// SecurityFix that this update is a security fix (that the comment includes `(* Security fix *)`)
|
||||
func (u Update) SecurityFix() bool { |
||||
return strings.Contains(u.Comment, securityFixStr) |
||||
} |
||||
|
||||
// ToChangeLog reformats the struct as the text for ChangeLog.txt output
|
||||
func (u Update) ToChangeLog() string { |
||||
return fmt.Sprintf("%s: %s.\n%s", u.Name, u.Action, u.Comment) |
||||
} |
||||
@ -0,0 +1,61 @@
|
||||
package changelog |
||||
|
||||
import ( |
||||
"os" |
||||
"strings" |
||||
"testing" |
||||
) |
||||
|
||||
func TestParse(t *testing.T) { |
||||
fh, err := os.Open("testdata/ChangeLog.txt") |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
defer fh.Close() |
||||
|
||||
e, err := Parse(fh) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
// Make sure we got all the entries
|
||||
expectedLen := 52 |
||||
if len(e) != expectedLen { |
||||
t.Errorf("expected %d entries; got %d", expectedLen, len(e)) |
||||
} |
||||
|
||||
// Make sure we got as many security fix entries as expected
|
||||
expectedSec := 34 |
||||
secCount := 0 |
||||
for i := range e { |
||||
if e[i].SecurityFix() { |
||||
secCount++ |
||||
} |
||||
} |
||||
if secCount != expectedSec { |
||||
t.Errorf("expected %d security fix entries; got %d", expectedSec, secCount) |
||||
} |
||||
|
||||
// Make sure we got as many individual updates as expected
|
||||
expectedUp := 597 |
||||
upCount := 0 |
||||
for i := range e { |
||||
upCount += len(e[i].Updates) |
||||
} |
||||
if upCount != expectedUp { |
||||
t.Errorf("expected %d updates across the entries; got %d", expectedUp, upCount) |
||||
} |
||||
|
||||
// Make sure the top comment of an entry is working
|
||||
foundWorkmanComment := false |
||||
expectedComment := "Thanks to Robby Workman for most of these updates." |
||||
for i := range e { |
||||
foundWorkmanComment = strings.Contains(e[i].Comment, expectedComment) |
||||
if foundWorkmanComment { |
||||
break |
||||
} |
||||
} |
||||
if !foundWorkmanComment { |
||||
t.Errorf("expected to find an Entry with comment %q", expectedComment) |
||||
} |
||||
} |
||||
File diff suppressed because it is too large
Load Diff
@ -1,2 +0,0 @@
|
||||
#0 2 * * * ~/opt/bin/ruby ~/bin/changelog_alphageek.rb > /dev/null || echo "$(date): failed aphageek" | mail -s "[slackagg] alphageek's changelog failed $(date +%D)" vbatts@hashbangbash.com |
||||
0 */2 * * * python ~/bin/changelog_http_poll.py >/dev/null || echo "$(date): failed to poll changelogs" | mail -s "[slackrss] changelog_http_poll failed $(date +%D)" vbatts@hashbangbash.com |
||||
Loading…
Reference in new issue