Update: new features for nix-template-rpm

- nix-template-rpm can now split the generated templates into
    a static part that goes into the nixpkgs tree
    a dynamic part that can be updated easily to track the rpm spec files
- add lookup mechanism for package names and package paths
- add mechanism to update existing nix-expression with new download files

pkgs/build-support/templaterpm/nix-template-rpm.py

@@ -4,6 +4,7 @@ import sys
 import os
 import subprocess
 import argparse
+import re
 import shutil
 import rpm
 import urlparse
@@ -14,8 +15,8 @@ import toposort
 
 
 
-class NixTemplateRPM(object):
+class SPECTemplate(object):
-  def __init__(self, specFilename, inputDir=None, maintainer="MAINTAINER"):
+  def __init__(self, specFilename, outputDir, inputDir=None, buildRootInclude=None, translateTable=None, repositoryDir=None, allPackagesDir=None, maintainer="MAINTAINER"):
     rpm.addMacro("buildroot","$out")
     rpm.addMacro("_libdir","lib")
     rpm.addMacro("_libexecdir","libexec")
@@ -24,22 +25,41 @@ class NixTemplateRPM(object):
     rpm.addMacro("_topdir","SPACER_DIR_FOR_REMOVAL")
     rpm.addMacro("_sourcedir","SOURCE_DIR_SPACER")
 
+    self.packageGroups = [ "ocaml", "python" ]
+
     ts = rpm.TransactionSet()
 
     self.specFilename = specFilename
     self.spec = ts.parseSpec(specFilename)
 
     self.inputDir = inputDir
+    self.buildRootInclude = buildRootInclude
+    self.repositoryDir = repositoryDir
+    self.allPackagesDir = allPackagesDir
     self.maintainer = maintainer
 
-    self.packageGroups = [ "ocaml", "python" ]
+    self.translateTable = translateTable
+
+    self.facts = self.getFacts()
+    self.key = self.getSelfKey()
+
+    tmpDir = os.path.join(outputDir, self.rewriteName(self.spec.sourceHeader['name']))
+    if self.translateTable != None:
+      self.relOutputDir = self.translateTable.path(self.key,tmpDir)
+    else:
+      self.relOutputDir = tmpDir
+
+    self.final_output_dir = os.path.normpath( self.relOutputDir )
+
+    if self.repositoryDir != None:
+      self.potential_repository_dir = os.path.normpath( os.path.join(self.repositoryDir,self.relOutputDir) )
 
 
 
   def rewriteCommands(self, string):
     string = string.replace('SPACER_DIR_FOR_REMOVAL/','')
     string = string.replace('SPACER_DIR_FOR_REMOVAL','')
-    string = '\n'.join(map(lambda line: ' '.join(map(lambda x: x.replace('SOURCE_DIR_SPACER/','${./')+'}' if x.startswith('SOURCE_DIR_SPACER/') else x, line.split(' '))), string.split('\n')))
+    string = '\n'.join(map(lambda line: ' '.join(map(lambda x: x.replace('SOURCE_DIR_SPACER/',('${./' if (self.buildRootInclude == None) else '${buildRoot}/usr/share/buildroot/SOURCES/'))+('}' if (self.buildRootInclude == None) else '') if x.startswith('SOURCE_DIR_SPACER/') else x, line.split(' '))), string.split('\n')))
     string = string.replace('\n','\n    ')
     string = string.rstrip()
     return string
@@ -85,18 +105,31 @@ class NixTemplateRPM(object):
     else:
       raise Exception("Unknown target")
       packages = []
+
     return packages
 
 
   def getBuildInputs(self,target=None):
-    return self.rewriteInputs(target,self.spec.sourceHeader['requires'])
+    inputs = self.rewriteInputs(target,self.spec.sourceHeader['requires'])
+    if self.translateTable != None:
+      return map(lambda x: self.translateTable.name(x), inputs)
+    else:
+      return inputs
 
-  def getSelf(self):
+  def getSelfKey(self):
     name = self.spec.sourceHeader['name']
     if len(name.split('-')) > 1 and name.split('-')[0] in self.packageGroups:
-      return self.rewriteInputs(name.split('-')[0], [self.spec.sourceHeader['name']])[0]
+      key = self.rewriteInputs(name.split('-')[0], [self.spec.sourceHeader['name']])[0]
     else:
-      return self.rewriteInputs(None, [self.spec.sourceHeader['name']])[0]
+      key = self.rewriteInputs(None, [self.spec.sourceHeader['name']])[0]
+    return key
+
+  def getSelf(self):
+    if self.translateTable != None:
+      return self.translateTable.name(self.key)
+    else:
+      return self.key
+
 
 
 
@@ -112,12 +145,34 @@ class NixTemplateRPM(object):
       shutil.copyfile(os.path.join(input_dir, filename), os.path.join(output_dir, filename))
 
 
+  def getFacts(self):
+    facts = {}
+    facts["name"] = self.rewriteName(self.spec.sourceHeader['name'])
+    facts["version"] = self.spec.sourceHeader['version']
+
+    facts["url"] = []
+    facts["sha256"] = []
+    sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ]
+    for url in sources:
+      p = subprocess.Popen(['nix-prefetch-url', url], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+      output, err = p.communicate()
+      sha256 = output[:-1] #remove new line
+      facts["url"].append(url)
+      facts["sha256"].append(sha256)
+
+    patches = [source for (source, _, flag) in self.spec.sources if flag==2]
+    if self.buildRootInclude == None:
+      facts["patches"] = map(lambda x: './'+x, patches)
+    else:
+      facts["patches"] = map(lambda x: '"${buildRoot}/usr/share/buildroot/SOURCES/'+x+'"', reversed(patches))
+
+    return facts
+
 
   @property
   def name(self):
-    out = 'stdenv.mkDerivation {\n'
+    out = '  name = "' + self.facts["name"] + '-' + self.facts["version"] + '";\n'
-    out += '  name = "' + self.rewriteName(self.spec.sourceHeader['name']) + '-' + self.spec.sourceHeader['version'] + '";\n'
+    out += '  version = "' + self.facts['version'] + '";\n'
-    out += '  version = "' + self.spec.sourceHeader['version'] + '";\n'
     return out
 
 
@@ -125,10 +180,7 @@ class NixTemplateRPM(object):
   def src(self):
     sources = [source for (source, _, flag) in self.spec.sources if flag==1 if urlparse.urlparse(source).scheme in ["http", "https"] ]
     out = ''
-    for url in sources:
+    for (url,sha256) in zip(self.facts['url'],self.facts['sha256']):
-      p = subprocess.Popen(['nix-prefetch-url', url], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-      output, err = p.communicate()
-      sha256 = output[:-1] #remove new line
       out += '  src = fetchurl {\n'
       out += '    url = "' + url + '";\n'
       out += '    sha256 = "' + sha256 + '";\n'
@@ -138,8 +190,7 @@ class NixTemplateRPM(object):
 
   @property
   def patch(self):
-    patches = [source for (source, _, flag) in self.spec.sources if flag==2]
+    out = '  patches = [ ' + ' '.join(self.facts['patches']) + ' ];\n'
-    out = '  patches = [ ' + ' '.join(map(lambda x: './'+x, patches)) + ' ];\n'
     return out
 
 
@@ -189,11 +240,29 @@ class NixTemplateRPM(object):
     return out
 
 
-
   def __str__(self):
-     head = '{stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n'
+    head = '{stdenv, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n'
-     body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ]
+    head += 'stdenv.mkDerivation {\n'
-     return head + '\n'.join(body)
+    body = [ self.name, self.src, self.patch, self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ]
+    return head + '\n'.join(body)
+
+
+  def getTemplate(self):
+    head = '{stdenv, buildRoot, fetchurl, ' + ', '.join(self.getBuildInputs("ALL")) + '}:\n\n'
+    head += 'let\n'
+    head += '  buildRootInput = (import "${buildRoot}/usr/share/buildroot/buildRootInput.nix") { fetchurl=fetchurl; buildRoot=buildRoot; };\n'
+    head += 'in\n\n'
+    head += 'stdenv.mkDerivation {\n'
+    head += '  inherit (buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+') name version src;\n'
+    head += '  patches = buildRootInput.'+self.rewriteName(self.spec.sourceHeader['name'])+'.patches ++ [];\n\n'
+    body = [ self.buildInputs, self.configure, self.build, self.ocamlExtra, self.install, self.meta ]
+    return head + '\n'.join(body)
+
+
+  def getInclude(self):
+    head = self.rewriteName(self.spec.sourceHeader['name']) + ' = {\n'
+    body = [ self.name, self.src, self.patch ]
+    return head + '\n'.join(body) + '};\n'
 
 
   def __cmp__(self,other):
@@ -203,8 +272,8 @@ class NixTemplateRPM(object):
       return -1
 
 
-  def callPackage(self, output_dir):
+  def callPackage(self):
-    callPackage = '  ' + self.getSelf() + ' = callPackage ' + os.path.relpath(output_dir) + ' {'
+    callPackage = '  ' + self.getSelf() + ' = callPackage ' + os.path.relpath(self.final_output_dir, self.allPackagesDir) + ' {'
     newline = False;
     for target in self.packageGroups:
       tmp = self.getBuildInputs(target)
@@ -219,25 +288,141 @@ class NixTemplateRPM(object):
 
 
 
-  def generateTemplate(self, outputDir):
+  def generateCombined(self):
-    output_dir = os.path.normpath( os.path.join(outputDir, self.rewriteName(self.spec.sourceHeader['name'])) )
+    if not os.path.exists(self.final_output_dir):
-    if not os.path.exists(output_dir):
+      os.makedirs(self.final_output_dir)
-      os.makedirs(output_dir)
 
     if self.inputDir != None:
-      self.copySources(self.inputDir, output_dir)
+      self.copySources(self.inputDir, self.final_output_dir)
-      self.copyPatches(self.inputDir, output_dir)
+      self.copyPatches(self.inputDir, self.final_output_dir)
 
-    nixfile = open(os.path.join(output_dir,'default.nix'), 'w')
+    nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w')
     nixfile.write(str(self))
     nixfile.close()
 
-    shutil.copyfile(self.specFilename, os.path.join(output_dir, os.path.basename(self.specFilename)))
+    shutil.copyfile(self.specFilename, os.path.join(self.final_output_dir, os.path.basename(self.specFilename)))
-
-    self.pkgCall = self.callPackage(output_dir)
 
 
 
+  def generateSplit(self):
+    if not os.path.exists(self.final_output_dir):
+      os.makedirs(self.final_output_dir)
+
+    nixfile = open(os.path.join(self.final_output_dir,'default.nix'), 'w')
+    nixfile.write(self.getTemplate())
+    nixfile.close()
+
+    return self.getInclude()
+
+
+
+
+
+
+class NixTemplate(object):
+  def __init__(self, nixfile):
+    self.nixfile = nixfile
+    self.original = { "name":None, "version":None, "url":None, "sha256":None, "patches":None }
+    self.update = { "name":None, "version":None, "url":None, "sha256":None, "patches":None }
+    self.matchedLines = {}
+
+    if os.path.isfile(nixfile):
+      with file(nixfile, 'r') as infile:
+        for (n,line) in enumerate(infile):
+          name = re.match(r'^\s*name\s*=\s*"(.*?)"\s*;\s*$', line)
+          version = re.match(r'^\s*version\s*=\s*"(.*?)"\s*;\s*$', line)
+          url = re.match(r'^\s*url\s*=\s*"?(.*?)"?\s*;\s*$', line)
+          sha256 = re.match(r'^\s*sha256\s*=\s*"(.*?)"\s*;\s*$', line)
+          patches = re.match(r'^\s*patches\s*=\s*(\[.*?\])\s*;\s*$', line)
+          if name != None and self.original["name"] == None:
+              self.original["name"] = name.group(1)
+              self.matchedLines[n] = "name"
+          if version != None and self.original["version"] == None:
+              self.original["version"] = version.group(1)
+              self.matchedLines[n] = "version"
+          if url != None and self.original["url"] == None:
+              self.original["url"] = url.group(1)
+              self.matchedLines[n] = "url"
+          if sha256 != None and self.original["sha256"] == None:
+              self.original["sha256"] = sha256.group(1)
+              self.matchedLines[n] = "sha256"
+          if patches != None and self.original["patches"] == None:
+              self.original["patches"] = patches.group(1)
+              self.matchedLines[n] = "patches"
+
+
+  def generateUpdated(self, nixOut):
+    nixTemplateFile = open(os.path.normpath(self.nixfile),'r')
+    nixOutFile = open(os.path.normpath(nixOut),'w')
+    for (n,line) in enumerate(nixTemplateFile):
+      if self.matchedLines.has_key(n) and self.update[self.matchedLines[n]] != None:
+        nixOutFile.write(line.replace(self.original[self.matchedLines[n]], self.update[self.matchedLines[n]], 1))
+      else:
+        nixOutFile.write(line)
+    nixTemplateFile.close()
+    nixOutFile.close()
+
+
+  def loadUpdate(self,orig):
+    if orig.has_key("name") and orig.has_key("version"):
+      self.update["name"] = orig["name"] + '-' + orig["version"]
+      self.update["version"] = orig["version"]
+    if orig.has_key("url") and orig.has_key("sha256") and len(orig["url"])>0:
+      self.update["url"] = orig["url"][0]
+      self.update["sha256"] = orig["sha256"][0]
+      for url in orig["url"][1:-1]:
+        sys.stderr.write("WARNING: URL has been dropped: %s\n" % url)
+    if orig.has_key("patches"):
+      self.update["patches"] = '[ ' + ' '.join(orig['patches']) + ' ]'
+
+
+class TranslationTable(object):
+  def __init__(self):
+    self.tablePath = {}
+    self.tableName = {}
+
+  def update(self, key, path, name=None):
+    self.tablePath[key] = path
+    if name != None:
+      self.tableName[key] = name
+
+  def readTable(self, tableFile):
+    with file(tableFile, 'r') as infile:
+      for line in infile:
+        match = re.match(r'^(.+?)\s+(.+?)\s+(.+?)\s*$', line)
+        if match != None:
+          if not self.tablePath.has_key(match.group(1)):
+            self.tablePath[match.group(1)] = match.group(2)
+          if not self.tableName.has_key(match.group(1)):
+            self.tableName[match.group(1)] = match.group(3)
+        else:
+          match = re.match(r'^(.+?)\s+(.+?)\s*$', line)
+          if not self.tablePath.has_key(match.group(1)):
+            self.tablePath[match.group(1)] = match.group(2)
+
+  def writeTable(self, tableFile):
+    outFile = open(os.path.normpath(tableFile),'w')
+    keys = self.tablePath.keys()
+    keys.sort()
+    for k in keys:
+      if self.tableName.has_key(k):
+        outFile.write( k + " " + self.tablePath[k] + " " + self.tableName[k] + "\n" )
+      else:
+        outFile.write( k + " " + self.tablePath[k] + "\n" )
+    outFile.close()
+
+  def name(self, key):
+   if self.tableName.has_key(key):
+     return self.tableName[key]
+   else:
+     return key
+
+  def path(self, key, orig):
+   if self.tablePath.has_key(key):
+     return self.tablePath[key]
+   else:
+     return orig
+
 
 
 
@@ -247,23 +432,65 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser(description="Generate .nix templates from RPM spec files")
     parser.add_argument("specs", metavar="SPEC", nargs="+", help="spec file")
     parser.add_argument("-o", "--output", metavar="OUT_DIR", required=True, help="output directory")
-    parser.add_argument("-i", "--input", metavar="IN_DIR", default=None, help="input directory")
+    parser.add_argument("-b", "--buildRoot", metavar="BUILDROOT_DIR", default=None, help="buildroot output directory")
-    parser.add_argument("-m", "--maintainer", metavar="MAINTAINER", required=True, help="package maintainer")
+    parser.add_argument("-i", "--inputSources", metavar="IN_DIR", default=None, help="sources input directory")
+    parser.add_argument("-m", "--maintainer", metavar="MAINTAINER", default="__NIX_MAINTAINER__", help="package maintainer")
+    parser.add_argument("-r", "--repository", metavar="REP_DIR", default=None, help="nix repository to compare output against")
+    parser.add_argument("-t", "--translate", metavar="TRANSLATE_TABLE", default=None, help="path of translation table for name and path")
+    parser.add_argument("-u", "--translateOut", metavar="TRANSLATE_OUT", default=None, help="output path for updated translation table")
+    parser.add_argument("-a", "--allPackages", metavar="ALL_PACKAGES", default=None, help="top level dir to call packages from")
     args = parser.parse_args()
 
+    allPackagesDir = os.path.normpath( os.path.dirname(args.allPackages) )
+    if not os.path.exists(allPackagesDir):
+      os.makedirs(allPackagesDir)
 
+    buildRootContent = {}
     nameMap = {}
 
+    newTable = TranslationTable()
+    if args.translate != None:
+      table = TranslationTable()
+      table.readTable(args.translate)
+      newTable.readTable(args.translate)
+    else:
+      table = None
+
     for specPath in args.specs:
       try:
         sys.stderr.write("INFO: generate nix file from: %s\n" % specPath)
-        spec = NixTemplateRPM(specPath, args.input, args.maintainer)
+
-        spec.generateTemplate(args.output)
+        spec = SPECTemplate(specPath, args.output, args.inputSources, args.buildRoot, table, args.repository, allPackagesDir, args.maintainer)
+        if args.repository != None:
+          if os.path.exists(os.path.join(spec.potential_repository_dir,'default.nix')):
+            nixTemplate = NixTemplate(os.path.join(spec.potential_repository_dir,'default.nix'))
+            nixTemplate.loadUpdate(spec.facts)
+            if not os.path.exists(spec.final_output_dir):
+              os.makedirs(spec.final_output_dir)
+            nixTemplate.generateUpdated(os.path.join(spec.final_output_dir,'default.nix'))
+          else:
+            sys.stderr.write("WARNING: Repository does not contain template: %s\n" % os.path.join(spec.potential_repository_dir,'default.nix'))
+            if args.buildRoot == None:
+              spec.generateCombined()
+            else:
+              buildRootContent[spec.key] = spec.generateSplit()
+        else:
+          if args.buildRoot == None:
+            spec.generateCombined()
+          else:
+            buildRootContent[spec.key] = spec.generateSplit()
+
+        newTable.update(spec.key,spec.relOutputDir,spec.getSelf())
         nameMap[spec.getSelf()] = spec
 
       except Exception, e:
         sys.stderr.write("ERROR: %s failed with:\n%s\n%s\n" % (specPath,e.message,traceback.format_exc()))
 
+    if args.translateOut != None:
+      if not os.path.exists(os.path.dirname(os.path.normpath(args.translateOut))):
+        os.makedirs(os.path.dirname(os.path.normpath(args.translateOut)))
+      newTable.writeTable(args.translateOut)
+
     graph = {}
     for k, v in nameMap.items():
       graph[k] = set(v.getBuildInputs("ALL"))
@@ -271,4 +498,21 @@ if __name__ == "__main__":
     sortedSpecs = toposort.toposort_flatten(graph)
     sortedSpecs = filter( lambda x: x in nameMap.keys(), sortedSpecs)
 
-    print '\n\n'.join(map(lambda x: x.pkgCall, map(lambda x: nameMap[x], sortedSpecs)))
+    allPackagesFile = open(os.path.normpath( args.allPackages ), 'w')
+    allPackagesFile.write( '\n\n'.join(map(lambda x: x.callPackage(), map(lambda x: nameMap[x], sortedSpecs))) )
+    allPackagesFile.close()
+
+    if args.buildRoot != None:
+      buildRootFilename = os.path.normpath( args.buildRoot )
+      if not os.path.exists(os.path.dirname(buildRootFilename)):
+        os.makedirs(os.path.dirname(buildRootFilename))
+      buildRootFile = open(buildRootFilename, 'w')
+      buildRootFile.write( "{ fetchurl, buildRoot }: {\n\n" )
+      keys = buildRootContent.keys()
+      keys.sort()
+      for k in keys:
+        buildRootFile.write( buildRootContent[k] + '\n' )
+      buildRootFile.write( "}\n" )
+      buildRootFile.close()
+
+