Browse Source

changelog: a slackware ChangeLog parser

and ability to export to feeds

Signed-off-by: Vincent Batts <vbatts@hashbangbash.com>
pull/4/head
Vincent Batts 6 years ago
parent
commit
8e97e3d16f
Signed by: vbatts
GPG Key ID: 10937E57733F1362
  1. 28
      LICENSE
  2. 15
      README
  3. 18
      README.md
  4. 62
      bin/changelog_alphageek.rb
  5. 194
      bin/changelog_http_poll.py
  6. 39
      bin/changelog_pyinotify.py
  7. 155
      bin/changelog_slackware-rss.rb
  8. 68
      bin/gen_changlog_rss.rb
  9. 51
      changelog/feeds.go
  10. 38
      changelog/feeds_test.go
  11. 124
      changelog/parse.go
  12. 61
      changelog/parse_test.go
  13. 1430
      changelog/testdata/ChangeLog.txt
  14. 2
      crontab

28
LICENSE

@ -0,0 +1,28 @@
Copyright (c) 2016 Vincent Batts, Raleigh, NC, USA
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software without
specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

15
README

@ -1,15 +0,0 @@
hack scripts
They accomplish going from ChangeLog.txt -> RSS feeds that folks can subscribed to.
Ultimately ending up at http://www.slackware.com/~vbatts/feeds/
These are a mess, but still work.
I wrote a utility called slack-utils (https://github.com/vbatts/slack-utils/)
with a ruby gem (https://rubygems.org/gems/slack-utils). The python script
(`./bin/changelog_http_poll.py`) is called by a crontab. Python walks the
changelog.txt, then fetches from the http mirror the modified time. If it http
has the new version, then run the ruby script (`./bin/gen_changlog_rss.rb`).
This is what parses the changelog and returns RSS. Which is then written to the
corresponding file for public consumption.

18
README.md

@ -0,0 +1,18 @@
# sl-feeds
This is for proccessing Slackware ChangeLog.txt -> RSS feeds that folks can
subscribed to.
Ultimately ending up at http://www.slackware.com/~vbatts/feeds/
## Usage
```bash
go get github.com/vbatts/sl-feeds
```
crontab like:
```
0 */2 * * * ~/bin/sl-feeds -q || echo "$(date): failed to poll changelogs" | mail -s "[slackrss] changelog_http_poll failed $(date +%D)" me@example.com
```

62
bin/changelog_alphageek.rb

@ -1,62 +0,0 @@
#!/home/vbatts/opt/bin/ruby
#require 'fileutils'
require 'logger'
require 'tempfile'
require 'stringio'
require 'rubygems'
require 'slackware'
require 'slackware/changelog/rss'
#include FileUtils
$LOG = Logger.new(STDERR)
$LOG.level = Logger::WARN
FEEDS_BASE_DIR = "/home/vbatts/public_html/feeds/"
#url = 'http://alphageek.dyndns.org/linux/slackware-packages/slack-13.1/ChangeLog.txt'
# Sun Feb 13 08:44:35 PST 2011
# new url
URL = 'http://alphageek.dyndns.org/mirrors/alphageek/slackware-%s/ChangeLog.txt'
VERSIONS = %w{ 14.0 14.1 }
def url(ver)
URL % ver
end
if ARGV.include?('-v')
$LOG.level = Logger::DEBUG
end
VERSIONS.each {|ver|
begin
#tmp_file = File.open("/tmp/vbatts/alpha_log-#{(rand*1000).to_i}.xxx", "w+")
tmp_file = Tempfile.new("alpha_log")
$LOG.debug('tmp_file') { tmp_file }
strio = StringIO.new()
$LOG.debug('created ') { strio }
buffer = `lynx -source #{url(ver)}`
$LOG.debug('buffer length') { buffer.length }
tmp_file.write(buffer)
tmp_file.flush
changelog = Slackware::ChangeLog.new(tmp_file.path)
changelog.parse
strio.write(changelog.to_rss(
:noimage => true,
:title => "alphageek's #{ver} ChangeLog",
:url => url(ver)))
ensure
strio.seek(0)
tmp_file.close
end
feed_file = File.open(FEEDS_BASE_DIR + "alphageek-#{ver}_ChangeLog.rss", "w+")
$LOG.debug('feed_file') { feed_file }
feed_file.write(strio.read())
feed_file.close
}

194
bin/changelog_http_poll.py

@ -1,194 +0,0 @@
#!/usr/bin/env python
# Mon Oct 17 08:25:29 PDT 2011
# copyright 2011 Vincent Batts, Vienna, VA, USA
# switching from an inotify watcher, to an http poll
# since what lands on connie.slackware.com usually doesn't go public
# immediately
import os
import sys
import glob
import time
from datetime import datetime
from datetime import timedelta
from time import mktime
import urllib2
import anydbm
DEFAULT_DB = os.path.join(os.getenv('HOME'), '.slackware_changelog.db')
DEFAULT_URL = "http://slackware.osuosl.org/"
SLACKWARE_DIR_PATH = "/mirrors/ftp.slackware.com/pub/slackware"
RSS_DIR_PATH = "/home/vbatts/public_html/feeds"
'''
slackware-12.2_ChangeLog.rss
/home/vbatts/public_html/feeds/slackware-10.1_patches_ChangeLog.rss
/home/vbatts/public_html/feeds/slackware-8.1_patches_ChangeLog.rss
>>> for i in c.slackware_versions(): print i
...
/mirrors/ftp.slackware.com/pub/slackware/slackware64-13.0/ChangeLog.txt
/mirrors/ftp.slackware.com/pub/slackware/slackware-8.1/ChangeLog.txt
/mirrors/ftp.slackware.com/pub/slackware/slackware64-13.37/ChangeLog.txt
/mirrors/ftp.slackware.com/pub/slackware/slackware-13.0/ChangeLog.txt
/mirrors/ftp.slackware.com/pub/sla
'''
def rss_files():
for item in glob.glob(RSS_DIR_PATH + "/*.rss"):
yield item
def rss_files_format(str):
if str.startswith(RSS_DIR_PATH + "/"):
str = str[len(RSS_DIR_PATH + "/"):]
if str.endswith(".rss"):
str = str[:-4]
str = str + '.txt'
return str.replace('_','/')
def rss_files_cleaned():
for i in rss_files():
yield rss_files_format(i)
def slackware_versions():
changes = glob.glob(SLACKWARE_DIR_PATH + "/*/ChangeLog.txt")
patches = glob.glob(SLACKWARE_DIR_PATH + "/*/patches/ChangeLog.txt")
for item in changes + patches:
yield item
def slackware_versions_format(str):
if str.startswith(SLACKWARE_DIR_PATH + "/"):
str = str[len(SLACKWARE_DIR_PATH + "/"):]
if str.endswith("/"):
str = str[:-1]
if str.startswith("/"):
str = str[1:]
if str.endswith(".txt"):
str = str[:-4]
return str.replace('/','_')
def slackware_versions_strip():
for i in slackware_versions():
yield i[len(SLACKWARE_DIR_PATH + "/"):]
def slackware_versions_rss():
for i in slackware_versions():
yield slackware_versions_format(i)
def process_changelog_rss(pathname):
if os.path.basename(pathname) == "ChangeLog.txt":
print "%f: proccessing %s" % (time.time(), pathname)
# XXX REPLACE ME!!
cmd = "/home/vbatts/opt/bin/ruby /home/vbatts/bin/gen_changlog_rss.rb %s" % pathname
print cmd
print os.system(cmd)
else:
print '[WARN] "%s" is not a ChangeLog.txt file' % pathname
def db_setup(name = DEFAULT_DB):
try:
return anydbm.open(name, 'c')
except:
return None
def db_teardown(db):
try:
return db.close()
except:
return None
def db_add_ts(db, key, val):
if type(val) == float:
db[key] = str(val)
if type(val) == datetime:
db[key] = str(unix_time(val))
return db[key]
def db_get_ts(db, key):
try:
return datetime.fromtimestamp(float(db[key]))
except KeyError:
return None
def unix_time(dt):
return mktime(dt.timetuple())+1e-6*dt.microsecond
def time_from_header(str):
return datetime.strptime(str, "%a, %d %b %Y %H:%M:%S %Z")
def get_remote_header(url, header):
try:
req = urllib2.Request(url)
resp = urllib2.urlopen(req)
return resp.headers.getheader(header)
except:
return None
def get_remote_time_str(url):
return get_remote_header(url,"last-modified")
def get_remote_time(url):
time_str = get_remote_time_str(url)
if time_str:
return time_from_header(time_str)
else:
return None
def get_local_time(path):
try:
time_flt = os.stat(path).st_mtime
return datetime.fromtimestamp(time_flt)
except:
return None
def main(args):
try:
db = db_setup()
if db == None:
print "ERROR: could not setup database at %s" % DEFAULT_DB
return 1
for i in slackware_versions_strip():
# i'm not going to worry about this file, right now
if i == 'slackware/ChangeLog.txt':
continue
rss_file_name = os.path.join(RSS_DIR_PATH,
slackware_versions_format(i) + ".rss")
rss_ts = get_local_time(rss_file_name)
curr_ts = get_local_time(os.path.join(SLACKWARE_DIR_PATH, i))
prev_ts = db_get_ts( db, "local_" + i)
# Go no further for this file
if curr_ts == prev_ts and os.path.exists(rss_file_name) and rss_ts > prev_ts:
print '[INFO] Local time of "%s" is same as the database has' % i
continue
db_add_ts( db, "local_" + i, curr_ts)
remote_ts = get_remote_time(DEFAULT_URL + i)
print '[INFO] inserting remote_%s: %s' % (i,remote_ts)
db_add_ts( db, "remote_" + i, remote_ts)
if prev_ts == None or (remote_ts - prev_ts) == timedelta(hours=7):
print '[INFO] local and remote ChangeLog times match'
if rss_ts == None:
print '[INFO] RSS file (%s) does not exist' % (rss_ts)
print '[INFO] Processing "%s"' % rss_file_name
process_changelog_rss(os.path.join(SLACKWARE_DIR_PATH, i))
elif prev_ts == None or rss_ts < prev_ts:
print '[INFO] RSS file (%s) is older than the ChangeLog (%s)' % (rss_ts, prev_ts)
print '[INFO] Processing "%s"' % rss_file_name
process_changelog_rss(os.path.join(SLACKWARE_DIR_PATH, i))
else:
print '[INFO] RSS seems current'
finally:
try:
os.wait()
except:
pass
db_teardown(db)
if __name__ == "__main__": sys.exit(main(sys.argv[1:]))

39
bin/changelog_pyinotify.py

@ -1,39 +0,0 @@
#!/usr/bin/env python
import os
import sys
import glob
import time
sys.path.insert(0, "/home/vbatts/opt/lib/python2.5/site-packages")
sys.path.insert(0, "/home/vbatts/opt/lib/python2.5")
import pyinotify
dir_path = "/mirrors/ftp.slackware.com/pub/slackware"
def process_changelog_rss(event):
if os.path.basename(event.pathname) == "ChangeLog.txt":
print "%f: proccessing %s" % (time.time(), event)
os.system("/home/vbatts/opt/bin/ruby /home/vbatts/bin/gen_changlog_rss.rb %s" % event.pathname)
def main(args):
wm = pyinotify.WatchManager()
notifier = pyinotify.Notifier(wm)
for dir in glob.glob(dir_path + "/*/"):
if os.path.exists(dir + "ChangeLog.txt"):
print "%f: Adding watch for %s" % (time.time(), dir)
wm.add_watch(dir, pyinotify.IN_MOVED_TO, rec=False, proc_fun=process_changelog_rss)
for dir in glob.glob(dir_path + "/*/patches/"):
print "%f: Adding watch for %s" % (time.time(), dir)
wm.add_watch(dir, pyinotify.IN_MOVED_TO, rec=False, proc_fun=process_changelog_rss)
#wm.add_watch("/home/vbatts/", pyinotify.IN_MOVED_TO, rec=False, proc_fun=process_changelog_rss)
notifier.loop()
if __name__ == "__main__": main(sys.argv[1:])

155
bin/changelog_slackware-rss.rb

@ -1,155 +0,0 @@
#!/usr/bin/env ruby
# Sun Jan 23 11:30:53 PST 2011
# Created by vbatts, vbatts@hashbangbash.com
$PROGRAM_NAME = File.basename(__FILE__)
require 'find'
require 'rubygems'
require 'ruby-prof'
require 'slackware'
require 'slackware/changelog/rss'
require 'rb-inotify'
BASE_URL = "http://slackware.osuosl.org/"
MIRROR_BASE_DIR = "/mirrors/ftp.slackware.com/pub/slackware/"
FEEDS_BASE_DIR = "/home/vbatts/public_html/feeds/"
RE_REPO_NAME = Regexp.new(/slackware(\d{2})?-(\d+\.\d+|current)\/(patches)?\/?.*/)
def generate_new_if_none
files = []
Find.find(MIRROR_BASE_DIR) {|file|
relative_name = file.sub(MIRROR_BASE_DIR, "")
if File.basename(file) == "ChangeLog.txt"
if not(relative_name.include?("zipslack"))
files << relative_name
Find.prune
end
end
# putting this check *after* the one above,
# lets us get the patches directories too
# while still getting a bit of speed (1.5s)
if relative_name.split("/").count > 2
Find.prune
end
}
puts "%f: watching %d changelogs" % [Time.now.to_f, files.count]
files.each {|file|
m = RE_REPO_NAME.match file
if m[3].nil?
file_name = "%sslackware%s-%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2]]
else
file_name = "%sslackware%s-%s_%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2], m[3]]
end
unless File.exist?(file_name)
c_file = MIRROR_BASE_DIR + file
changelog = Slackware::ChangeLog.new(c_file, :version => m[2])
changelog.opts[:arch] = m[1] unless m[1].nil?
if m[3].nil?
changelog.opts[:url] = "%sslackware%s-%s/ChangeLog.txt" % [BASE_URL, m[1], m[2]]
feed = File.open( "%sslackware%s-%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2]], "w+")
else
changelog.opts[:url] = "%sslackware%s-%s/%s/ChangeLog.txt" % [BASE_URL, m[1], m[2], m[3]]
feed = File.open( "%sslackware%s-%s_%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2], m[3]], "w+")
end
changelog.parse
puts "%f: Making a first feed: %s" % [Time.now.to_f, feed.path]
feed << changelog.to_rss
feed.close
changelog = nil
end
}
end
def run_notifier
n = INotify::Notifier.new
dirs = Dir.glob(MIRROR_BASE_DIR + "*")
dirs.concat(Dir.glob(MIRROR_BASE_DIR + "*/patches/"))
dirs.each {|dir|
next unless File.exist?(File.join(dir, "ChangeLog.txt"))
puts "%f: working with %s" % [Time.now.to_f, dir]
n.watch(dir, :moved_to) {|mfile|
file_name = mfile.absolute_name
if File.basename(file_name) == "ChangeLog.txt"
puts "%f: looking into %s" % [Time.now.to_f, file_name]
match_data = RE_REPO_NAME.match(file_name)
unless match_data.nil?
changelog = Slackware::ChangeLog.new(file_name, :version => match_data[2])
changelog.opts[:arch] = match_data[1] unless match_data[1].nil?
if match_data[3].nil?
changelog.opts[:url] = "%sslackware%s-%s/ChangeLog.txt" % [
BASE_URL,
match_data[1],
match_data[2]
]
feed = File.open( "%sslackware%s-%s_ChangeLog.rss" % [
FEEDS_BASE_DIR,
match_data[1],
match_data[2]
], "w+")
else
changelog.opts[:url] = "%sslackware%s-%s/%s/ChangeLog.txt" % [
BASE_URL,
match_data[1],
match_data[2],
match_data[3]
]
feed = File.open( "%sslackware%s-%s_%s_ChangeLog.rss" % [
FEEDS_BASE_DIR,
match_data[1],
match_data[2],
match_data[3]
], "w+")
end
begin
changelog.parse
rescue StandardError => ex
puts "%f: %s" % [Time.now.to_f, ex.message]
puts "%f: %s" % [Time.now.to_f, file_name]
next
end
puts "%f: parsed %s to %s" % [Time.now.to_f, file_name, feed.path]
feed << changelog.to_rss
feed.close
changelog = nil
end
end
}
}
begin
n.run
rescue Interrupt
end
end
## Main
#generate_new_if_none()
begin
RubyProf.start
run_notifier()
ensure
result = RubyProf.stop
RubyProf.measure_mode = RubyProf::PROCESS_TIME
RubyProf.measure_mode = RubyProf::WALL_TIME
RubyProf.measure_mode = RubyProf::CPU_TIME
#RubyProf.measure_mode = RubyProf::ALLOCATIONS
#RubyProf.measure_mode = RubyProf::MEMORY
#RubyProf.measure_mode = RubyProf::GC_RUNS
#RubyProf.measure_mode = RubyProf::GC_TIME
output_file_name = File.join(ENV["HOME"],"%s-%s%s" % [Time.now.to_i.to_s,File.basename(__FILE__),".log"])
output_file = File.open(output_file_name, "w+")
printer = RubyProf::FlatPrinter.new(result)
printer.print(output_file,0)
puts "%f: %s written" % [Time.now.to_f, output_file_name]
output_file.close
end

68
bin/gen_changlog_rss.rb

@ -1,68 +0,0 @@
#!/home/vbatts/opt/bin/ruby
require 'logger'
$log = Logger.new(STDERR)
$log.level = Logger::DEBUG
# put this in a loader function, because the
# rss library is SOO SLOW to load. we don't want to load it,
# if the script is going to fail early.
def load_libs()
require 'rubygems'
require 'slackware'
require 'slackware/changelog/rss'
require 'rb-inotify'
end
BASE_URL = "http://slackware.osuosl.org/"
FEEDS_BASE_DIR = "/home/vbatts/public_html/feeds/"
RE_REPO_NAME = Regexp.new(/slackware(\d{2})?-(\d+\.\d+|current)\/(patches)?\/?.*/)
def gen_file(file)
m = RE_REPO_NAME.match file
if m[3].nil?
file_name = "%sslackware%s-%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2]]
else
file_name = "%sslackware%s-%s_%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2], m[3]]
end
if File.exist?(file_name)
if File.mtime(file) < File.mtime(file_name)
printf("%f: INFO: %s is newer than %s\n", Time.now, file, file_name)
end
end
changelog = Slackware::ChangeLog.new(file) #, :version => m[2])
opts = Hash.new
opts[:arch] = m[1] unless m[1].nil?
if m[3].nil?
opts[:url] = "%sslackware%s-%s/ChangeLog.txt" % [BASE_URL, m[1], m[2]]
feed = File.open( "%sslackware%s-%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2]], "w+")
else
opts[:url] = "%sslackware%s-%s/%s/ChangeLog.txt" % [BASE_URL, m[1], m[2], m[3]]
feed = File.open( "%sslackware%s-%s_%s_ChangeLog.rss" % [FEEDS_BASE_DIR, m[1], m[2], m[3]], "w+")
end
changelog.parse
printf("%f: INFO: generating feed: %s\n", Time.now.to_f, feed.path)
feed << changelog.to_rss(opts)
feed.close
changelog = nil
end
if ARGV.count == 0
$log.error("#{Time.now}: ERROR: ChangeLog.txt files must be passed\n")
exit(2)
else
load_libs()
for file in ARGV
if File.exist?(file)
gen_file(file)
else
$log.warn("#{Time.now}: WARN: #{file} does not exist\n")
end
end
end
# vim: set sts=2 sw=2 et ai:

51
changelog/feeds.go

@ -0,0 +1,51 @@
package changelog
import (
"fmt"
"time"
"github.com/gorilla/feeds"
)
// ToFeed produces a github.com/gorilla/feeds.Feed that can be written to Atom or Rss
func ToFeed(link string, entries []Entry) (*feeds.Feed, error) {
var newestEntryTime time.Time
var oldestEntryTime time.Time
for _, e := range entries {
if e.Date.After(newestEntryTime) {
newestEntryTime = e.Date
}
if e.Date.Before(oldestEntryTime) {
oldestEntryTime = e.Date
}
}
feed := &feeds.Feed{
Title: "",
Link: &feeds.Link{Href: link},
Description: "Generated ChangeLog.txt feeds by sl-feeds (github.com/vbatts/sl-feeds)",
Created: oldestEntryTime,
Updated: newestEntryTime,
}
feed.Items = make([]*feeds.Item, len(entries))
for i, e := range entries {
feed.Items[i] = &feeds.Item{
Created: e.Date,
Link: &feeds.Link{Href: ""},
Description: e.ToChangeLog(),
}
updateWord := "updates"
if len(e.Updates) == 1 {
updateWord = "update"
}
if e.SecurityFix() {
feed.Items[i].Title = fmt.Sprintf("%d %s. Including a %s!", len(e.Updates), updateWord, securityFixStr)
} else {
feed.Items[i].Title = fmt.Sprintf("%d %s.", len(e.Updates), updateWord)
}
}
return feed, nil
}

38
changelog/feeds_test.go

@ -0,0 +1,38 @@
package changelog
import (
"io/ioutil"
"os"
"testing"
)
func TestFeed(t *testing.T) {
fh, err := os.Open("testdata/ChangeLog.txt")
if err != nil {
t.Fatal(err)
}
defer fh.Close()
e, err := Parse(fh)
if err != nil {
t.Fatal(err)
}
f, err := ToFeed("http://slackware.osuosl.org/slackware64-current/ChangeLog.txt", e)
if err != nil {
t.Fatal(err)
}
rss, err := f.ToRss()
if err != nil {
t.Fatal(err)
}
//println(rss)
if len(rss) == 0 {
t.Error("rss output is empty")
}
if err := f.WriteRss(ioutil.Discard); err != nil {
t.Error(err)
}
}

124
changelog/parse.go

@ -0,0 +1,124 @@
package changelog
import (
"bufio"
"fmt"
"io"
"regexp"
"strings"
"time"
)
const (
dividerStr = `+--------------------------+`
securityFixStr = `(* Security fix *)`
dayPat = `^(Mon|Tue|Wed|Thu|Fri|Sat|Sun)\s(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s.*\d{4}$`
updatePat = `^([a-z].*/.*): (Added|Rebuilt|Removed|Updated|Upgraded)\.$`
)
var (
dayReg = regexp.MustCompile(dayPat)
updateReg = regexp.MustCompile(updatePat)
)
// Parse takes in a slackware ChangeLog.txt and returns its collections of Entries
func Parse(r io.Reader) ([]Entry, error) {
buf := bufio.NewReader(r)
entries := []Entry{}
curEntry := Entry{}
var curUpdate *Update
for {
line, err := buf.ReadString('\n')
if err != nil && err != io.EOF {
return nil, err
}
isEOF := err == io.EOF
trimmedline := strings.TrimSuffix(line, "\n")
if trimmedline == dividerStr {
if curUpdate != nil {
curEntry.Updates = append(curEntry.Updates, *curUpdate)
curUpdate = nil
}
entries = append(entries, curEntry)
if isEOF {
break
}
curEntry = Entry{}
} else if dayReg.MatchString(trimmedline) {
// this date means it is the beginning of an entry
t, err := time.Parse(time.UnixDate, trimmedline)
if err != nil {
return nil, err
}
curEntry.Date = t
} else if updateReg.MatchString(trimmedline) {
// match on whether this is an update line
if curUpdate != nil {
curEntry.Updates = append(curEntry.Updates, *curUpdate)
curUpdate = nil
}
m := updateReg.FindStringSubmatch(trimmedline)
curUpdate = &Update{
Name: m[1],
Action: m[2],
}
} else if curUpdate != nil && strings.HasPrefix(trimmedline, " ") {
curUpdate.Comment = curUpdate.Comment + line
} else {
// Everything else is a comment on the Entry
curEntry.Comment = curEntry.Comment + line
}
if isEOF {
break
}
}
return entries, nil
}
// Entry is an section of updates (or release comments) in a ChangeLog.txt
type Entry struct {
Date time.Time
Comment string
Updates []Update
}
// SecurityFix is whether an update in this ChangeLog Entry includes a SecurityFix
func (e Entry) SecurityFix() bool {
for _, u := range e.Updates {
if u.SecurityFix() {
return true
}
}
return false
}
// ToChangeLog reformats the struct as the text for ChangeLog.txt output
func (e Entry) ToChangeLog() string {
str := e.Date.Format(time.UnixDate) + "\n"
if strings.Trim(e.Comment, " \n") != "" {
str = str + e.Comment
}
for _, u := range e.Updates {
str = str + u.ToChangeLog()
}
return str
}
// Update is a package or component that is updated in a ChangeLog Entry
type Update struct {
Name string
Action string
Comment string
}
// SecurityFix that this update is a security fix (that the comment includes `(* Security fix *)`)
func (u Update) SecurityFix() bool {
return strings.Contains(u.Comment, securityFixStr)
}
// ToChangeLog reformats the struct as the text for ChangeLog.txt output
func (u Update) ToChangeLog() string {
return fmt.Sprintf("%s: %s.\n%s", u.Name, u.Action, u.Comment)
}

61
changelog/parse_test.go

@ -0,0 +1,61 @@
package changelog
import (
"os"
"strings"
"testing"
)
func TestParse(t *testing.T) {
fh, err := os.Open("testdata/ChangeLog.txt")
if err != nil {
t.Fatal(err)
}
defer fh.Close()
e, err := Parse(fh)
if err != nil {
t.Fatal(err)
}
// Make sure we got all the entries
expectedLen := 52
if len(e) != expectedLen {
t.Errorf("expected %d entries; got %d", expectedLen, len(e))
}
// Make sure we got as many security fix entries as expected
expectedSec := 34
secCount := 0
for i := range e {
if e[i].SecurityFix() {
secCount++
}
}
if secCount != expectedSec {
t.Errorf("expected %d security fix entries; got %d", expectedSec, secCount)
}
// Make sure we got as many individual updates as expected
expectedUp := 597
upCount := 0
for i := range e {
upCount += len(e[i].Updates)
}
if upCount != expectedUp {
t.Errorf("expected %d updates across the entries; got %d", expectedUp, upCount)
}
// Make sure the top comment of an entry is working
foundWorkmanComment := false
expectedComment := "Thanks to Robby Workman for most of these updates."
for i := range e {
foundWorkmanComment = strings.Contains(e[i].Comment, expectedComment)
if foundWorkmanComment {
break
}
}
if !foundWorkmanComment {
t.Errorf("expected to find an Entry with comment %q", expectedComment)
}
}

1430
changelog/testdata/ChangeLog.txt vendored

File diff suppressed because it is too large Load Diff

2
crontab

@ -1,2 +0,0 @@
#0 2 * * * ~/opt/bin/ruby ~/bin/changelog_alphageek.rb > /dev/null || echo "$(date): failed aphageek" | mail -s "[slackagg] alphageek's changelog failed $(date +%D)" vbatts@hashbangbash.com
0 */2 * * * python ~/bin/changelog_http_poll.py >/dev/null || echo "$(date): failed to poll changelogs" | mail -s "[slackrss] changelog_http_poll failed $(date +%D)" vbatts@hashbangbash.com
Loading…
Cancel
Save