[cvs] / gkb / gkb.py  

cvs: gkb/gkb.py


1 : tmcnulty 1.1 #!/usr/bin/python
2 :     #
3 :     # GKB - GNU Kernel Builder
4 : tmcnulty 1.2 # Copyright (C) 2003-2004 Tobias McNulty, Mark Guertin
5 : tmcnulty 1.1 #
6 :     # This program is free software; you can redistribute it and/or modify
7 :     # it under the terms of the GNU General Public License as published by
8 :     # the Free Software Foundation; either version 2 of the License, or
9 :     # (at your option) any later version.
10 :     #
11 :     # This program is distributed in the hope that it will be useful,
12 :     # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 :     # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 :     # GNU General Public License for more details.
15 :     #
16 :     # You should have received a copy of the GNU General Public License
17 :     # along with this program; if not, write to the Free Software
18 :     # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
19 :     #
20 :     # Ported from PHP version Copyright (c) Tobias McNulty 2000-2003
21 :    
22 :     import sys, os, ConfigParser
23 :     import xml.parsers.expat
24 :     import md5
25 :     from ftplib import FTP
26 :     from commands import getoutput
27 :     from shutil import copy, move
28 :     from string import split
29 :     from urllib2 import urlopen
30 :     from ClientForm import ParseResponse
31 :    
32 :     debug=0
33 : tmcnulty 1.4 clean=0
34 : tmcnulty 1.1 verbose=1
35 :    
36 :     # parse host-specific configuration information from gkb.cfg
37 :     config = ConfigParser.ConfigParser()
38 :     config.readfp(open('gkb-host.cfg'))
39 :     buildroot=config.get("options","buildroot")
40 :     host=config.get("options","host")
41 :     passwd=config.get("options","passwd")
42 :    
43 :     # this one is passed verbatim before the make command. might be useful for alternate toolchains, etc
44 :     # example: premake="HOST=powerpc-unknown-linux-gnu"
45 :     premake=config.get("options","premake")
46 :     # make options, passed verbatim after make command
47 :     makeopts=config.get("options","makeopts")
48 :    
49 :     # parse manager-related configuration from gkb-manager.cfg
50 :     config.readfp(open('gkb-manager.cfg'))
51 :     msite=config.get('options','msite')
52 :    
53 :     # these values will likely stay fairly static across different builds
54 :     patchdir="%s/patches" % buildroot # full path to patch file dir
55 :     configdir="%s/configs" % buildroot # path to where configs are to be stored
56 :     logdir="%s/logs" % buildroot # the logs will go here
57 :     bindir="%s/bin" % buildroot # the directory that binaries built will be sent for pkging
58 :     workdir="%s/work" % buildroot # work directory, where builds take place
59 :    
60 :     # setup the global dicts to hold build data, imported from xml
61 :     mastertrees={}
62 :     builds={}
63 :    
64 :     def printverbose(myoutput):
65 :     """Supporting method to output info to stdout if verbosity level is set"""
66 :     if verbose==1:
67 :     print myoutput
68 :    
69 :     def log(text,kinfo=0):
70 :     if kinfo:
71 :     printverbose(kinfo["name"] + " : " + text)
72 :     else:
73 :     printverbose("gkb : " + text)
74 :    
75 :     def deathbyerror(mymsg):
76 :     """Supporting method to print error to stdout and return system error -1 on exit"""
77 :     print mymsg
78 :     sys.exit(-1)
79 :    
80 :     def verifydir(mydir,kinfo=0):
81 :     """Supporting method to verify a given exists, if not it will create it"""
82 :     log("verifying directory " + mydir,kinfo)
83 :     if not os.path.isdir(mydir):
84 :     log("%s doesn't exist, creating" % mydir,kinfo)
85 :     os.mkdir(mydir)
86 :     return mydir
87 :    
88 :     def chdir(mydir,kinfo):
89 :     verifydir(mydir,kinfo)
90 :     log("entering "+mydir,kinfo)
91 :     os.chdir(mydir)
92 :    
93 :     def verifyfile(myfile,kinfo):
94 :     """Supporting method to verify file exists, if not it will exit with error"""
95 :     if not os.path.isfile(myfile):
96 :     deathbyerror("%s : cannot find file expected at %s, exiting" % (kinfo["name"],myfile))
97 :     return myfile
98 :    
99 :     def md5sum(fileobj):
100 :     """calculates the md5sum for fileobj and returns it in hexadecimal"""
101 :     md = md5.new()
102 :     data = fileobj.read(8192)
103 :     while (data):
104 :     md.update(data)
105 :     data = fileobj.read(8192)
106 :    
107 :     digest = md.hexdigest()
108 :     return digest
109 :    
110 :     def krn_querymgr(command,kinfo):
111 :     """queries the build host manager with a variety of commands, such as checkout, checkin, etc."""
112 :     version=krn_localversion(kinfo)
113 :     log("querying distribution site manager with command '%s' (kernel version=%s)" % (command,version),kinfo)
114 :     result=getoutput("wget --quiet --output-document=- \"%s/manager.php?cmd=%s&host=%s&pass=%s&build=%s&version=%s\"" % (msite,command,host,passwd,kinfo["name"],version))
115 :     log("result: '"+result+"'",kinfo)
116 :     return result=="1"
117 :    
118 :     def krn_localversion(kinfo):
119 :     """Checks the version of the local source tree specified in kinfo"""
120 :     version=getoutput("""awk -F '=' '/^VERSION/{v=$2} /^PATCHLEVEL/{p=$2} /^SUBLEVEL/{s=$2} /^EXTRAVERSION/{e=$2} END { printf("%s.%s.%s%s\\n", v, p, s, e) }' """ + kinfo["workdir"] + """/Makefile | sed "s/ //g" """)
121 :     return version
122 :    
123 :     def gkb_build(root,kinfo):
124 :     """performs actual kernel builds given root, kinfo (tuple containing relevant data) , premake and makeopts"""
125 :    
126 :     # make sure needed directoriess (for this build) exist, if not create them
127 :    
128 :     verifydir("%s/%s" % (logdir, kinfo["name"]),kinfo)
129 :     verifydir("%s/%s" % (patchdir, kinfo["name"]),kinfo)
130 :    
131 :     # sync the source to make sure we are up to date ...
132 :     log("calling gkb_getsource",kinfo)
133 :     gkb_getsource(kinfo)
134 :    
135 :     # now we'll go into the work
136 :     chdir(kinfo["workdir"],kinfo)
137 :    
138 :     # check for patches and apply if necessary
139 :     gkb_patch(kinfo)
140 :    
141 :     if kinfo["type"] == "kernel24":
142 :     krn_build24(kinfo)
143 :     elif kinfo["type"] == "kernel26":
144 :     krn_build24(kinfo) #run kernel24 for now, change later
145 :    
146 :     # Gerk comment:
147 :     # Do we need to change this with 24/26? we probably don't ... I'd rather see us define the list of make targets
148 :     # i.e. "dep","clean","vmlinux","modules","modules_install" or "clean","all","modules_install" for 2.6
149 :     # also of note, make all is not what we will always want with 2.6 kernels ... i.e. zImage.prep, zImage.chrp, etc
150 :     # also for backward compat they will continue to support calling things individual .. the makefile just does
151 :     # this for us with 'all'
152 :    
153 :    
154 :     def krn_build24(kinfo):
155 :     """build a kernel, version 2.4.x"""
156 :    
157 :     if krn_querymgr("checkout",kinfo):
158 :     myversion=krn_localversion(kinfo)
159 :    
160 :     # fetch and cp the config file to work/.config
161 :     krn_config(kinfo)
162 :    
163 :     # **Note** : we set the preprocessing command to premake inline instead
164 :     # of globally as it is only needed in this target
165 :     gkb_runmake("oldconfig",kinfo,premake+" /bin/cat %s/newlines | " % buildroot,makeopts)
166 :    
167 :     # give option to only repackage for testing purposes, comment out clean=1 at top of this file to use this feature
168 :     if clean==1:
169 :     gkb_runmake("clean", kinfo, premake, makeopts)
170 :    
171 :     gkb_runmake("dep", kinfo, premake, makeopts)
172 :    
173 :     gkb_runmake(kinfo["binname"], kinfo, premake, makeopts)
174 :    
175 :     # We should check to see if binary built ok, if not bail out
176 :     mybindir=bindir+"/linux-"+kinfo["name"]+"-"+myversion
177 :     verifydir(mybindir,kinfo)
178 :     verifydir(mybindir+"/boot",kinfo)
179 :    
180 :     kbinloc=kinfo["workdir"]+"/"+kinfo["binpath"]+"/"+kinfo["binname"]
181 :     if verifyfile(kbinloc,kinfo):
182 :     # the binary exists, so let's cp it to bin...
183 :     copy(kbinloc,mybindir+"/boot/"+kinfo["binname"]+"-"+myversion)
184 :     else:
185 :     # the binary is not there, inform user and bail out with error
186 :     deathbyerror("%s : %s is not present, assuming build failure and exiting. See log for details." % (kinfo["name"], kbinloc))
187 :    
188 :     # now that we know he binary built, let's continue
189 :     gkb_runmake("modules",kinfo, premake, makeopts)
190 :    
191 :     # **Note** : we prepend the INSTALL_MOD_PATH to the makeopts inline instead
192 :     # of globally as it is only needed in this target
193 :     gkb_runmake("modules_install", kinfo, premake, "INSTALL_MOD_PATH=%s %s" % (mybindir, makeopts))
194 :    
195 :     chdir(bindir,kinfo)
196 : tmcnulty 1.4 archive_name = "linux-%s-%s.tar.bz2" % (kinfo["name"], myversion)
197 : tmcnulty 1.3
198 :     log("compressing binary archive "+archive_name,kinfo)
199 :    
200 : tmcnulty 1.4 if os.system("tar cjf "+archive_name+" "+os.path.basename(mybindir)):
201 :     deathbyerror("%s : failed to `tar cjf %s`" % (kinfo["name"], archive_name))
202 : tmcnulty 1.1
203 :     os.system("rm -rf %s" % mybindir)
204 :    
205 : tmcnulty 1.3 krn_upload(archive_name,myversion,kinfo)
206 :    
207 : tmcnulty 1.1 krn_querymgr("checkin",kinfo)
208 :    
209 :     def krn_build26(kinfo):
210 :     """build a kernel, version 2.6"""
211 :    
212 :     # source get routines
213 :     def gkb_getsource(kinfo):
214 :     """method to perform source sync on demand, currenty only supports rsync, but others can be added"""
215 :    
216 :     method=mastertrees[kinfo["mastertree"]]["method"]
217 :    
218 :     if method == "rsync":
219 :     get_rsync(kinfo)
220 :     elif method == "wget":
221 :     get_wget(kinfo)
222 :     elif method == "vanilla":
223 :     get_vanilla(kinfo)
224 :    
225 :     def get_rsync(kinfo):
226 :     """sync the source using rsync"""
227 :     args=mastertrees[kinfo["mastertree"]]["args"]
228 :    
229 :     if clean==1:
230 :     syncoptions="rsync -azv --delete"
231 :     else:
232 :     syncoptions="rsync -azv"
233 :    
234 :     syncline=args+" "+kinfo["workdir"]
235 :    
236 :     log("running rsync : %s %s" % (syncoptions, syncline),kinfo)
237 :     if os.system("%s %s > %s/%s/rsync.log 2>&1" % (syncoptions, syncline, logdir, kinfo["name"])):
238 :     deathbyerror("%s : sync failed, tried %s. See log for details." % (kinfo["name"], syncline) )
239 :    
240 :     # Gerk comment:
241 :     # This above might cause problems ... it (I think) relies on anything coming from stderr to tell
242 :     # if it has in fact died... we should test this out. If this is the case me might need to make
243 :     # a stub of some sorts to handle the build/error return process...
244 :    
245 :    
246 :     def get_wget(kinfo):
247 :     #needs work
248 :     """sync the source using wget and a tar.bz2"""
249 :     args=mastertrees[kinfo["mastertree"]]["args"]
250 :    
251 :     log("fetching source archive " % args,kinfo)
252 :     mysourcefile="%s/%s.tar.bz2" % (kinfo["workdir"],kinfo["name"])
253 :    
254 :     if os.system("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):
255 :     deathbyerror("%s : unable to download configfile, aborting." % (kinfo["name"]))
256 :    
257 :     log("decompressing source file",kinfo)
258 :     if os.system("tar xjf " % (myconfigfile,msite,kinfo["name"])):
259 :     deathbyerror("%s : unable to decompress source file, aborting." % (kinfo["name"]))
260 :    
261 :     def get_vanilla(kinfo):
262 :     args=mastertrees[kinfo["mastertree"]]["args"]
263 :    
264 :     log("fetching source archive %s" % args)
265 :     mysourcefile="%s/%s.tar.bz2" % (workdir,kinfo["name"])
266 :    
267 :     if os.system("wget -c --output-document=%s %s" % (mysourcefile,args)):
268 :     deathbyerror("%s : unable to download source file %s, aborting." % (kinfo["name"], args))
269 :    
270 :     log("decompressing source file %s" % mysourcefile,kinfo)
271 :    
272 :     #save the current working ectory
273 :     cwd=getoutput("pwd")
274 :    
275 :     chdir(workdir,kinfo)
276 :     pfd=os.popen("tar vxjf %s" % mysourcefile,"r")
277 :     trash=data=pfd.read(255)
278 :    
279 :     while trash:
280 :     trash=pfd.read(4096)
281 :    
282 :     if pfd.close():
283 :     deathbyerror("%s : unable to decompress source file, aborting." % (kinfo["name"]))
284 :    
285 :     fnames=split(data)
286 :     name=fnames[0]
287 :    
288 :     os.system("rm -rf %s" % kinfo["workdir"])
289 :     move(work + "/" + dirname, kinfo["workdir"])
290 :    
291 :     #restore the previous working ectory
292 :     chdir(cwd,kinfo)
293 :    
294 :     def gkb_patch(kinfo):
295 :     """method that applies patch found in kinfo config files for running kernel build"""
296 :     if kinfo["patches"]:
297 :     patches=split(kinfo["patches"],";")
298 :    
299 :     for patch in patches:
300 :     log("downloading patch %s" % patch,kinfo)
301 :    
302 :     #set a name for this patch file
303 :     mypatchfile="%s/%s/%s.patch" % (patchdir,kinfo["name"],patch)
304 :    
305 :     # download pathfile or bail
306 :     if os.system("wget --quiet --output-document=%s %s/patches/%s/%s" % (mypatchfile,msite,kinfo["name"],patch)):
307 :     deathbyerror("%s : unable to download patchfile %s, aborting." % (kinfo["name"],patch))
308 :    
309 :     # we have a patch file, apply it or bail
310 :     log("perfoming patch with %s" % patch,kinfo)
311 :     patchcommand="patch -p1 < %s > %s/%s/patch-%s.log 2>&1" % (mypatchfile,logdir,kinfo["name"],patch)
312 :     log("using %s from %s" % (patchcommand,kinfo["workdir"]),kinfo)
313 :     chdir(kinfo["workdir"],kinfo)
314 :     if os.system(patchcommand):
315 :     deathbyerror("%s : patchfile %s failed, aborting. See patch log for details." % (kinfo["name"],patch))
316 :     else:
317 :     log("no patchfiles, continuing",kinfo)
318 :    
319 :     def krn_config(kinfo):
320 :     """method to fetch and place config file for running kernel build"""
321 :    
322 :     if kinfo["config"]==1:
323 :     log("fetching config file",kinfo)
324 :     myconfigfile="%s/%s.config" % (configdir,kinfo["name"])
325 :    
326 :     if os.system("wget --quiet --output-document=%s %s/configs/%s" % (myconfigfile,msite,kinfo["name"])):
327 :     deathbyerror("%s : unable to download configfile, aborting." % (kinfo["name"]))
328 :    
329 :     log("copying config file to %s/.config" % kinfo["workdir"],kinfo)
330 :     copy(verifyfile(myconfigfile,kinfo),"%s/.config" % kinfo["workdir"])
331 :    
332 :     # Methods to support build()
333 :     def gkb_runmake(command, kinfo, premake, makeopts):
334 :     """Supporting method for build() ... a stub to run a make target and auto log it, given make target (command) and name (kernel name)"""
335 :     log("running make %s" % command,kinfo)
336 :     if os.system("%s make %s %s > %s/%s/make-%s.log 2>&1" % (premake, makeopts, command, logdir, kinfo["name"], command)):
337 :     deathbyerror("%s : unable to run make %s, aborting." % (kinfo["name"],command))
338 :    
339 :     def gkb_parsexml(name):
340 :     """parse the xml build file into mastertrees and builds"""
341 :     def start_element(name, attrs):
342 :     #print "In start_element: %s %s" % (name, attrs)
343 :     if name=="mastertree":
344 :     mastertrees[attrs["name"]]=attrs
345 :     elif name=="build":
346 :     builds[attrs["name"]]=attrs
347 :    
348 :     p = xml.parsers.expat.ParserCreate()
349 :     p.StartElementHandler = start_element
350 :     p.ParseFile(open(name))
351 : tmcnulty 1.3
352 :     def krn_upload(file, version, kinfo):
353 :     """upload the indicated file to the distribution site (kernel archives)"""
354 :     forms = ParseResponse(urlopen(msite+"/fileupload.html"))
355 :     form = forms[0]
356 :    
357 :     form["host"] = host
358 :     form["pass"] = passwd
359 :     form["build"] = kinfo["name"]
360 :     form["version"] = version
361 :    
362 :     form.add_file(open(file), None, os.path.basename(file))
363 :    
364 :     log("uploading kernel version "+version+" to "+msite,kinfo)
365 :     # form.click() returns a urllib2.Request object
366 :     # (see HTMLForm.click.__doc__ if you don't have urllib2)
367 :     response2 = urlopen(form.click("cmd"))
368 :    
369 :     #print response2.geturl()
370 :     #print response2.info() # headers
371 :     #print response2.read() # body
372 :     log("response: " + response2.read(),kinfo)
373 :    
374 :     response2.close()
375 :    
376 : tmcnulty 1.1 def main():
377 :     """ main program gets executed here """
378 :    
379 :     # get our working ectory
380 :     root=getoutput("pwd")
381 :    
382 :     today=getoutput("date +%D")
383 :     buildtime=getoutput("date +'%R:%S %Z'")
384 :    
385 :     print "GKB started %s %s" % (today,buildtime)
386 :    
387 :     # verify the existence important directories, and create if necessary
388 :     verifydir(logdir)
389 :     verifydir(configdir)
390 :     verifydir(patchdir)
391 :     verifydir(bindir)
392 :     verifydir("%s/work" % buildroot) # build dir, make sure it exists
393 :    
394 :     # download the build jobs from the master site
395 :     if os.system("wget --quiet --output-document=gkb.xml \"%s/manager.php?cmd=getjobs&host=%s&pass=%s\"" % (msite,host,passwd)):
396 :     deathbyerror("Unable to download build config from master site %s, aborting." % msite)
397 :    
398 :     # sets up 'mastertrees' and 'builds' dicts
399 :     gkb_parsexml('gkb.xml')
400 :    
401 :     for bdict in builds.values():
402 :     myworkdir="%s/%s" % (workdir,bdict["mastertree"])
403 :     bdict["workdir"]=myworkdir
404 :    
405 :     # for now just call the build
406 :     gkb_build(root, bdict)
407 :    
408 :     endtime=getoutput("date +'%R:%S %Z'")
409 :    
410 :     print "GKB finished %s %s" % (today,endtime)
411 :    
412 :     # and finally, call the mainloop to execute
413 :     main()

Tobias McNulty

Powered by ViewCVS 1.0-dev
(Powered by ViewCVS)

ViewCVS and CVS Help