1 '''
2 Whole-cell helper functions
3
4 Author: Jonathan Karr, jkarr@stanford.edu
5 Affiliation: Covert Lab, Department of Bioengineering, Stanford University
6 Last updated: 2012-07-17
7 '''
8
9 from __future__ import unicode_literals
10 from BeautifulSoup import BeautifulStoneSoup
11 from copy import deepcopy
12 from dateutil.tz import tzlocal
13 from django.contrib.auth.models import User
14 from django.core import serializers
15 from django.core.exceptions import ValidationError, ObjectDoesNotExist
16 from django.core.urlresolvers import reverse
17 from django.db import IntegrityError
18 from django.db.models import get_app, get_models, get_model
19 from django.db.models.fields import AutoField, BigIntegerField, IntegerField, PositiveIntegerField, PositiveSmallIntegerField, SmallIntegerField, BooleanField, NullBooleanField, DecimalField, FloatField, CharField, CommaSeparatedIntegerField, EmailField, FilePathField, GenericIPAddressField, IPAddressField, SlugField, URLField, TextField, DateField, DateTimeField, TimeField, NOT_PROVIDED
20 from django.db.models.fields.related import OneToOneField, RelatedObject, ManyToManyField, ForeignKey
21 from django.db.models.query import EmptyQuerySet
22 from django.http import Http404, HttpResponse, HttpResponseBadRequest
23 from django.shortcuts import render_to_response, get_object_or_404
24 from django.template import Context, RequestContext, loader
25 from django.template.loader import get_template
26 from django.utils import simplejson
27 from django.utils.html import strip_tags
28 from django.utils.text import capfirst
29 from odict import odict
30 from openpyxl import Workbook, load_workbook
31 from openpyxl.cell import Cell, get_column_letter
32 from openpyxl.shared.date_time import SharedDate, CALENDAR_WINDOWS_1900
33 from openpyxl.style import NumberFormat, Border, Color, HashableObject, Protection, Alignment
34 from openpyxl.writer.styles import StyleWriter
35 from public.templatetags.templatetags import ceil
36 from public.models import Entry, Species, SpeciesComponent, Reference, Chromosome, format_list_html, format_evidence, Evidence, EvidencedEntryData
37 from public.models import EntryData, EntryBooleanData, EntryCharData, EntryFloatData, EntryPositiveFloatData, EntryTextData
38 from pygments.filter import Filter
39 from pygments.formatters import HtmlFormatter
40 from pygments.style import Style
41 from pygments.token import Token
42 from StringIO import StringIO
43 from subprocess import Popen, PIPE
44 from xml.dom.minidom import Document
45 import datetime
46 import htmlentitydefs
47 import inspect
48 import math
49 import os
50 import pygments
51 import re
52 import settings
53 import sys
54 import tempfile
55 import time
56 import xhtml2pdf.pisa as pisa
57
59 H = 1.0079
60 He = 4.0026
61 Li = 6.941
62 Be = 9.0122
63 B = 10.811
64 C = 12.0107
65 N = 14.0067
66 O = 15.9994
67 F = 18.9984
68 Ne = 20.1797
69 Na = 22.9897
70 Mg = 24.305
71 Al = 26.9815
72 Si = 28.0855
73 P = 30.9738
74 S = 32.065
75 Cl = 35.453
76 Ar = 39.948
77 K = 39.0983
78 Ca = 40.078
79 Sc = 44.9559
80 Ti = 47.867
81 V = 50.9415
82 Cr = 51.9961
83 Mn = 54.938
84 Fe = 55.845
85 Co = 58.9332
86 Ni = 58.6934
87 Cu = 63.546
88 Zn = 65.39
89 Ga = 69.723
90 Ge = 72.64
91 As = 74.9216
92 Se = 78.96
93 Br = 79.904
94 Kr = 83.8
95 Rb = 85.4678
96 Sr = 87.62
97 Y = 88.9059
98 Zr = 91.224
99 Nb = 92.9064
100 Mo = 95.94
101 Tc = 98
102 Ru = 101.07
103 Rh = 102.9055
104 Pd = 106.42
105 Ag = 107.8682
106 Cd = 112.411
107 In = 114.818
108 Sn = 118.71
109 Sb = 121.76
110 Te = 127.6
111 I = 126.9045
112 Xe = 131.293
113 Cs = 132.9055
114 Ba = 137.327
115 La = 138.9055
116 Ce = 140.116
117 Pr = 140.9077
118 Nd = 144.24
119 Pm = 145
120 Sm = 150.36
121 Eu = 151.964
122 Gd = 157.25
123 Tb = 158.9253
124 Dy = 162.5
125 Ho = 164.9303
126 Er = 167.259
127 Tm = 168.9342
128 Yb = 173.04
129 Lu = 174.967
130 Hf = 178.49
131 Ta = 180.9479
132 W = 183.84
133 Re = 186.207
134 Os = 190.23
135 Ir = 192.217
136 Pt = 195.078
137 Au = 196.9665
138 Hg = 200.59
139 Tl = 204.3833
140 Pb = 207.2
141 Bi = 208.9804
142 Po = 209
143 At = 210
144 Rn = 222
145 Fr = 223
146 Ra = 226
147 Ac = 227
148 Th = 232.0381
149 Pa = 231.0359
150 U = 238.0289
151 Np = 237
152 Pu = 244
153 Am = 243
154 Cm = 247
155 Bk = 247
156 Cf = 251
157 Es = 252
158 Fm = 257
159 Md = 258
160 No = 259
161 Lr = 262
162 Rf = 261
163 Db = 262
164 Sg = 266
165 Bh = 264
166 Hs = 277
167 Mt = 268
168
169
171 single_dna = {
172 'A': 15400,
173 'C': 7200,
174 'G': 11500,
175 'T': 8700,
176 }
177 single_rna = {
178 'A': 15400,
179 'C': 7200,
180 'G': 11500,
181 'U': 9900,
182 }
183 pairwise_dna = {
184 'A': {'A': 27400, 'C': 21200, 'G': 25000, 'T': 28000},
185 'C': {'A': 21200, 'C': 14600, 'G': 18000, 'T': 15200},
186 'G': {'A': 25200, 'C': 17600, 'G': 21600, 'T': 20000},
187 'T': {'A': 23400, 'C': 16200, 'G': 19000, 'T': 16800},
188 }
189 pairwise_rna = {
190 'A': {'A': 27400, 'C': 21000, 'G': 25000, 'U': 24000},
191 'C': {'A': 21000, 'C': 14200, 'G': 17800, 'U': 16200},
192 'G': {'A': 25200, 'C': 17400, 'G': 21600, 'U': 21200},
193 'U': {'A': 24600, 'C': 17200, 'G': 20000, 'U': 19600},
194 }
195
196 '''
197 dipeptide instability weight value (DIWV)
198 1. http://ca.expasy.org/tools/protparam-doc.html
199 2. Kunchur Guruprasad, B.V.Bhasker Reddy and Madhusudan W.Pandit
200 (1990). Correlation between stability of a protein and its
201 dipeptide composition: a novel approach for predicting in vivo
202 stability of a protein from its primary sequence. Protein
203 Engineering 4(2):155-161.
204 '''
206 value = {
207 "A": {"A": 1.00, "R": 1.00, "N": 1.00, "D": -7.49, "C": 44.94, "Q": 1.00, "E": 1.00, "G": 1.00, "H": -7.49, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": 1.00, "P": 20.26, "S": 1.00, "T": 1.00, "W": 1.00, "Y": 1.00, "V": 1.00, },
208 "R": {"A": 1.00, "R": 58.28, "N": 13.34, "D": 1.00, "C": 1.00, "Q": 20.26, "E": 1.00, "G": -7.49, "H": 20.26, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": 1.00, "P": 20.26, "S": 44.94, "T": 1.00, "W": 58.28, "Y": -6.54, "V": 1.00, },
209 "N": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 1.00, "C": -1.88, "Q": -6.54, "E": 1.00, "G": -14.03, "H": 1.00, "I": 44.94, "L": 1.00, "K": 24.68, "M": 1.00, "F": -14.03, "P": -1.88, "S": 1.00, "T": -7.49, "W": -9.37, "Y": 1.00, "V": 1.00, },
210 "D": {"A": 1.00, "R": -6.54, "N": 1.00, "D": 1.00, "C": 1.00, "Q": 1.00, "E": 1.00, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": -7.49, "M": 1.00, "F": -6.54, "P": 1.00, "S": 20.26, "T": -14.03, "W": 1.00, "Y": 1.00, "V": 1.00, },
211 "C": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 20.26, "C": 1.00, "Q": -6.54, "E": 1.00, "G": 1.00, "H": 33.60, "I": 1.00, "L": 20.26, "K": 1.00, "M": 33.60, "F": 1.00, "P": 20.26, "S": 1.00, "T": 33.60, "W": 24.68, "Y": 1.00, "V": -6.54, },
212 "Q": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 20.26, "C": -6.54, "Q": 20.26, "E": 20.26, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": -6.54, "P": 20.26, "S": 44.94, "T": 1.00, "W": 1.00, "Y": -6.54, "V": -6.54, },
213 "E": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 20.26, "C": 44.94, "Q": 20.26, "E": 33.60, "G": 1.00, "H": -6.54, "I": 20.26, "L": 1.00, "K": 1.00, "M": 1.00, "F": 1.00, "P": 20.26, "S": 20.26, "T": 1.00, "W": -14.03, "Y": 1.00, "V": 1.00, },
214 "G": {"A": -7.49, "R": 1.00, "N": -7.49, "D": 1.00, "C": 1.00, "Q": 1.00, "E": -6.54, "G": 13.34, "H": 1.00, "I": -7.49, "L": 1.00, "K": -7.49, "M": 1.00, "F": 1.00, "P": 1.00, "S": 1.00, "T": -7.49, "W": 13.34, "Y": -7.49, "V": 1.00, },
215 "H": {"A": 1.00, "R": 1.00, "N": 24.68, "D": 1.00, "C": 1.00, "Q": 1.00, "E": 1.00, "G": -9.37, "H": 1.00, "I": 44.94, "L": 1.00, "K": 24.68, "M": 1.00, "F": -9.37, "P": -1.88, "S": 1.00, "T": -6.54, "W": -1.88, "Y": 44.94, "V": 1.00, },
216 "I": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 1.00, "C": 1.00, "Q": 1.00, "E": 44.94, "G": 1.00, "H": 13.34, "I": 1.00, "L": 20.26, "K": -7.49, "M": 1.00, "F": 1.00, "P": -1.88, "S": 1.00, "T": 1.00, "W": 1.00, "Y": 1.00, "V": -7.49, },
217 "L": {"A": 1.00, "R": 20.26, "N": 1.00, "D": 1.00, "C": 1.00, "Q": 33.60, "E": 1.00, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": -7.49, "M": 1.00, "F": 1.00, "P": 20.26, "S": 1.00, "T": 1.00, "W": 24.68, "Y": 1.00, "V": 1.00, },
218 "K": {"A": 1.00, "R": 33.60, "N": 1.00, "D": 1.00, "C": 1.00, "Q": 24.68, "E": 1.00, "G": -7.49, "H": 1.00, "I": -7.49, "L": -7.49, "K": 1.00, "M": 33.60, "F": 1.00, "P": -6.54, "S": 1.00, "T": 1.00, "W": 1.00, "Y": 1.00, "V": -7.49, },
219 "M": {"A": 13.34, "R": -6.54, "N": 1.00, "D": 1.00, "C": 1.00, "Q": -6.54, "E": 1.00, "G": 1.00, "H": 58.28, "I": 1.00, "L": 1.00, "K": 1.00, "M": -1.88, "F": 1.00, "P": 44.94, "S": 44.94, "T": -1.88, "W": 1.00, "Y": 24.68, "V": 1.00, },
220 "F": {"A": 1.00, "R": 1.00, "N": 1.00, "D": 13.34, "C": 1.00, "Q": 1.00, "E": 1.00, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": -14.03, "M": 1.00, "F": 1.00, "P": 20.26, "S": 1.00, "T": 1.00, "W": 1.00, "Y": 33.60, "V": 1.00, },
221 "P": {"A": 20.26, "R": -6.54, "N": 1.00, "D": -6.54, "C": -6.54, "Q": 20.26, "E": 18.38, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": 1.00, "M": -6.54, "F": 20.26, "P": 20.26, "S": 20.26, "T": 1.00, "W": -1.88, "Y": 1.00, "V": 20.26, },
222 "S": {"A": 1.00, "R": 20.26, "N": 1.00, "D": 1.00, "C": 33.60, "Q": 20.26, "E": 20.26, "G": 1.00, "H": 1.00, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": 1.00, "P": 44.94, "S": 20.26, "T": 1.00, "W": 1.00, "Y": 1.00, "V": 1.00, },
223 "T": {"A": 1.00, "R": 1.00, "N": -14.03, "D": 1.00, "C": 1.00, "Q": -6.54, "E": 20.26, "G": -7.49, "H": 1.00, "I": 1.00, "L": 1.00, "K": 1.00, "M": 1.00, "F": 13.34, "P": 1.00, "S": 1.00, "T": 1.00, "W": -14.03, "Y": 1.00, "V": 1.00, },
224 "W": {"A": -14.03, "R": 1.00, "N": 13.34, "D": 1.00, "C": 1.00, "Q": 1.00, "E": 1.00, "G": -9.37, "H": 24.68, "I": 1.00, "L": 13.34, "K": 1.00, "M": 24.68, "F": 1.00, "P": 1.00, "S": 1.00, "T": -14.03, "W": 1.00, "Y": 1.00, "V": -7.49, },
225 "Y": {"A": 24.68, "R": -15.91, "N": 1.00, "D": 24.68, "C": 1.00, "Q": 1.00, "E": -6.54, "G": -7.49, "H": 13.34, "I": 1.00, "L": 1.00, "K": 1.00, "M": 44.94, "F": 1.00, "P": 13.34, "S": 1.00, "T": -7.49, "W": -9.37, "Y": 13.34, "V": 1.00, },
226 "V": {"A": 1.00, "R": 1.00, "N": 1.00, "D": -14.03, "C": 1.00, "Q": 1.00, "E": 1.00, "G": -7.49, "H": 1.00, "I": 1.00, "L": 1.00, "K": -1.88, "M": 1.00, "F": 1.00, "P": 20.26, "S": 1.00, "T": -7.49, "W": 1.00, "Y": -6.54, "V": 1.00, },
227 }
228
245
253
262
271
273 for field in model._meta.fields + model._meta.many_to_many:
274 if isinstance(field, (ForeignKey, ManyToManyField)) and not (isinstance(field, OneToOneField) and field.rel.parent_link) and issubclass(field.rel.to, Entry):
275 if issubclass(referenced_model, field.rel.to):
276 return True
277
278 for field in model._meta.fields + model._meta.many_to_many:
279 if isinstance(field, (ForeignKey, ManyToManyField)) and not (isinstance(field, OneToOneField) and field.rel.parent_link) and issubclass(field.rel.to, Entry):
280 if (field.rel.to not in checked_models) and is_model_referenced(field.rel.to, referenced_model, checked_models + [model]):
281 return True
282 return False
283
291
300
308
316
324
326 app_ = __import__('public')
327 admins_ = getattr(app_, 'admin')
328 if hasattr(admins_, str + 'Admin'):
329 return getattr(admins_, str + 'Admin')
330 return None
331
333 unordered_fields = []
334 for field in model._meta.fields + model._meta.many_to_many:
335 if not (field.name in ['species', 'model_type'] or (isinstance(field, OneToOneField) and field.rel.parent_link)) and (not field.auto_created or auto_created):
336 unordered_fields.append(field)
337
338 ordered_fields = []
339 for field in model._meta.field_list:
340 if not metadata and field in ['id', 'created_user', 'last_updated_user']:
341 continue
342
343 field = model._meta.get_field_by_name(field)[0]
344 if not field.auto_created or auto_created:
345 ordered_fields.append(field)
346
347 if metadata and (len(unordered_fields) != len(ordered_fields) or set(unordered_fields) != set(ordered_fields)):
348 print str(set(unordered_fields).difference(ordered_fields))
349 print str(set(ordered_fields).difference(unordered_fields))
350 raise Http404
351
352 return ordered_fields
353
355 fields = []
356 for field in model._meta.fields + model._meta.many_to_many:
357 if not (field.name == 'model_type' or (isinstance(field, OneToOneField) and field.rel.parent_link)):
358 fields.append(field.name)
359 return fields
360
362 return inspect.isclass(cls) and issubclass(cls, EntryData)
363
365 return inspect.getmembers(sys.modules['public.models'], is_entrydata_model)
366
367 -def getEntry(species_wid = None, wid = None):
368 try:
369 species_id = Species.objects.values('id').get(wid = species_wid)['id']
370 except ObjectDoesNotExist:
371 return None
372
373 try:
374 if species_wid == wid:
375 return Species.objects.get(wid=wid)
376 else:
377 tmp = SpeciesComponent.objects.get(species__id = species_id, wid=wid)
378 return getModel(tmp.model_type).objects.select_related(depth=2).get(id=tmp.id)
379 except ObjectDoesNotExist:
380 return None
381
383 return strip_tags(unicode(BeautifulStoneSoup(s, convertEntities=BeautifulStoneSoup.ALL_ENTITIES)).replace(" ", " "))
384
385 '''
386 Element order preserved
387 From http://www.peterbe.com/plog/uniqifiers-benchmark
388 '''
390 seen = set()
391 return [x for x in seq if x not in seen and not seen.add(x)]
392
393 '''
394 Element order not preserved
395 From http://www.peterbe.com/plog/uniqifiers-benchmark
396 '''
398 return {}.fromkeys(seq).keys()
399
400 -def render_queryset_to_response(request = [], queryset = EmptyQuerySet(), models = [], templateFile = '', data = {}, species_wid=None):
401 format = request.GET.get('format', 'html')
402
403 if species_wid is not None and species_wid != '':
404 species = Species.objects.get(wid=species_wid)
405 else:
406 species = Species.objects.all()
407 if len(species) > 0:
408 species = species[0]
409 else:
410 species = None
411
412 data['species'] = species
413 data['queryset'] = queryset
414 data['request'] = request
415 data['queryargs'] = {}
416 for key, val in request.GET.iterlists():
417 data['queryargs'][key] = val
418
419 if format == 'html':
420 data['is_pdf'] = False
421 data['pdfstyles'] = ''
422 data['species_list'] = Species.objects.all()
423 data['modelmetadatas'] = getModelsMetadata(SpeciesComponent, request.user.is_anonymous())
424 data['modelnames'] = getObjectTypes(SpeciesComponent, request.user.is_anonymous())
425 data['last_updated_date'] = datetime.datetime.fromtimestamp(os.path.getmtime(settings.TEMPLATE_DIRS[0] + '/' + templateFile))
426
427 return render_to_response(templateFile, data, context_instance = RequestContext(request))
428 elif format == 'bib':
429 response = HttpResponse(
430 write_bibtex(species, queryset),
431 mimetype = "application/x-bibtex; charset=UTF-8",
432 content_type = "application/x-bibtex; charset=UTF-8")
433 response['Content-Disposition'] = "attachment; filename=data.bib"
434 elif format == 'json':
435 objects = []
436 for obj in queryset:
437 objDict = convert_modelobject_to_stdobject(obj, request.user.is_anonymous())
438 objDict['model'] = obj.__class__.__name__
439 objects.append(objDict)
440
441 now = datetime.datetime.now(tzlocal())
442 json = odict()
443 json['title'] = '%s WholeCellKB' % species.name
444 json['comments'] = 'Generated by %s on %s at %s' % ('WholeCellKB', now.isoformat(), settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}))
445 json['copyright'] = '%s %s' % (now.year, 'Covert Lab, Department of Bioengineering, Stanford University')
446 json['data'] = objects
447 response = HttpResponse(
448 simplejson.dumps(json, indent=2, ensure_ascii=False, encoding='utf-8'),
449 mimetype = "application/json; charset=UTF-8",
450 content_type = "application/json; charset=UTF-8")
451 response['Content-Disposition'] = "attachment; filename=data.json"
452 elif format == 'pdf':
453 data['is_pdf'] = True
454 data['pdfstyles'] = ''
455 data['species_list'] = Species.objects.all()
456 data['modelmetadatas'] = getModelsMetadata(SpeciesComponent, request.user.is_anonymous())
457 data['modelnames'] = getObjectTypes(SpeciesComponent, request.user.is_anonymous())
458
459 for fileName in ['styles', 'styles.print', 'styles.pdf']:
460 f = open(settings.STATICFILES_DIRS[0] + '/public/css/' + fileName + '.css', 'r')
461 data['pdfstyles'] += f.read()
462 f.close()
463
464 response = HttpResponse(
465 mimetype = 'application/pdf',
466 content_type = 'application/pdf')
467 response['Content-Disposition'] = "attachment; filename=data.pdf"
468
469 template = get_template(templateFile)
470
471 pdf = pisa.CreatePDF(
472 src = template.render(Context(data)),
473 dest = response)
474
475 if not pdf.err:
476 return response
477 return Http404
478 elif format == 'xlsx':
479
480 wb = writeExcel(species, queryset, models, request.user.is_anonymous())
481
482
483 result = StringIO()
484 wb.save(filename = result)
485
486
487 response = HttpResponse(
488 result.getvalue(),
489 mimetype = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
490 content_type = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
491 response['Content-Disposition'] = "attachment; filename=data.xlsx"
492 elif format == 'xml':
493 doc = Document()
494
495 now = datetime.datetime.now(tzlocal())
496 comment = doc.createComment('\n%s WholeCellKB\nGenerated by %s on %s at %s\n%s %s %s\n' % (
497 species.name,
498 'WholeCellKB', now.isoformat(),
499 settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}),
500 html_to_ascii('©'), now.year, 'Covert Lab, Department of Bioengineering, Stanford University',
501 ))
502 doc.appendChild(comment)
503
504 objects = doc.createElement('objects')
505 doc.appendChild(objects)
506
507 for obj in queryset:
508 objects.appendChild(convert_modelobject_to_xml(obj, doc, request.user.is_anonymous()))
509
510 response = HttpResponse(
511 doc.toprettyxml(indent=" "*2, encoding = 'utf-8'),
512 mimetype = "application/xml; charset=UTF-8",
513 content_type = "application/xml; charset=UTF-8")
514 response['Content-Disposition'] = "attachment; filename=data.xml"
515 else:
516 t = loader.get_template('public/error.html')
517 data['type'] = 500
518 data['message'] = '"%s" is not a supported export format.' % format
519 c = Context(data)
520 response = HttpResponseBadRequest(
521 t.render(c),
522 mimetype = 'text/html; charset=UTF-8',
523 content_type = 'text/html; charset=UTF-8')
524
525 return response
526
527 -def writeExcel(species, queryset, modelArr, is_user_anonymous):
528
529 modelDict = {}
530 model_types = []
531 objectDict = {}
532 for model in modelArr:
533 model_types.append(model.__name__)
534 modelDict[model.__name__] = model
535 objectDict[model.__name__] = []
536 model_types.sort()
537
538
539 for obj in queryset:
540 objectDict[obj.__class__.__name__].append(obj)
541
542
543 wb = Workbook()
544 wb.remove_sheet(wb.get_active_sheet())
545
546
547 now = datetime.datetime.now(tzlocal())
548 wb.properties.creator = 'Generated by WholeCellKB on %s at %s' % (
549 now.isoformat(),
550 settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}),
551 )
552 wb.properties.last_modified_by = wb.properties.creator
553 wb.properties.created = now
554 wb.properties.modified = now
555 wb.properties.title = '%s WholeCellKB' % species.name
556 wb.properties.subject = wb.properties.title
557 wb.properties.description = wb.properties.title
558 wb.properties.keywords = 'biology, systems, database, knowledge base'
559 wb.properties.category = ''
560 wb.properties.company = 'Covert Lab, Department of Bioengineering, Stanford University'
561 wb.properties.excel_base_date = CALENDAR_WINDOWS_1900
562
563
564 ws = wb.create_sheet(title = 'Contents')
565 ws.default_column_dimension.auto_size = True
566 ws.default_row_dimension.height = 15
567
568 headers = (
569 (wb.properties.title, 18, True),
570 ('Generated by WholeCellKB on %s at %s' % (
571 now.strftime("%Y-%m-%d"),
572 settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}),
573 ), 10, False),
574 ('%s %s %s' % (html_to_ascii('©'), now.year, wb.properties.company), 10, False),
575 )
576 for i in range(len(headers)):
577 ws.merge_cells(start_row=i, start_column=0, end_row=i, end_column=1)
578 cell = ws.cell(row=i, column=0)
579 cell.set_value_explicit(value = headers[i][0], data_type = Cell.TYPE_STRING)
580 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT
581 cell.style.font.name = 'Arial'
582 cell.style.font.size = headers[i][1]
583 cell.style.font.bold = headers[i][2]
584 cell.style.alignment.wrap_text = True
585 cell.style.alignment.horizontal = Alignment.HORIZONTAL_CENTER
586 cell.style.fill.fill_type = 'solid'
587 cell.style.fill.start_color.index = Colors.GREY1
588
589 ws.row_dimensions[i + 1].height = headers[i][1] + 7
590
591 cell = ws.cell(row=len(headers) + 1, column=0)
592 cell.set_value_explicit(value = 'Table', data_type = Cell.TYPE_STRING)
593 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT
594 cell.style.font.name = 'Arial'
595 cell.style.font.size = 10
596 cell.style.font.bold = True
597 cell.style.alignment.wrap_text = True
598 cell.style.alignment.horizontal = Alignment.HORIZONTAL_RIGHT
599 cell.style.borders.bottom.color.index = Colors.BLACK
600 cell.style.borders.bottom.border_style = Border.BORDER_THIN
601
602 cell = ws.cell(row=len(headers) + 1, column=1)
603 cell.set_value_explicit(value = 'Title', data_type = Cell.TYPE_STRING)
604 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT
605 cell.style.font.name = 'Arial'
606 cell.style.font.size = 10
607 cell.style.font.bold = True
608 cell.style.alignment.wrap_text = True
609 cell.style.alignment.horizontal = Alignment.HORIZONTAL_LEFT
610 cell.style.borders.bottom.color.index = Colors.BLACK
611 cell.style.borders.bottom.border_style = Border.BORDER_THIN
612
613 ws.freeze_panes = str('A%s' % (len(headers) + 3))
614 ws.column_dimensions['A'].width = 8
615 ws.column_dimensions['B'].width = 75
616
617 for i in range(len(model_types)):
618 iRow = i + len(headers) + 2
619 model = modelDict[model_types[i]]
620
621 cell = ws.cell(row=iRow, column=0)
622 cell.set_value_explicit(value = '%d.' % (i+1), data_type = Cell.TYPE_STRING)
623 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT
624 cell.style.font.name = 'Arial'
625 cell.style.font.size = 10
626 cell.style.alignment.wrap_text = True
627 cell.style.alignment.horizontal = Alignment.HORIZONTAL_RIGHT
628 cell.hyperlink = '#\'%s\'!%s' % (model._meta.verbose_name_plural[:31], 'A1')
629
630 cell = ws.cell(row=iRow, column=1)
631 cell.set_value_explicit(value = model._meta.verbose_name_plural, data_type = Cell.TYPE_STRING)
632 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT
633 cell.style.font.name = 'Arial'
634 cell.style.font.size = 10
635 cell.style.alignment.wrap_text = True
636 cell.style.alignment.horizontal = Alignment.HORIZONTAL_LEFT
637 cell.hyperlink = '#\'%s\'!%s' % (model._meta.verbose_name_plural[:31], 'A1')
638
639
640 for model_type in model_types:
641 model = modelDict[model_type]
642 fields = getModelDataFields(model, metadata=not is_user_anonymous)
643
644 iCol = -1
645 iCol_id = None
646 iCol_wid = None
647 for header, subheaders in get_excel_headers(model, is_user_anonymous=is_user_anonymous):
648 if header == 'ID':
649 iCol_id = iCol + 1
650 elif header == 'WID':
651 iCol_wid = iCol + 1
652 iCol += max(1, len(subheaders))
653
654
655 ws = wb.create_sheet(title = model._meta.verbose_name_plural[:31])
656 ws.freeze_panes = str(get_column_letter(iCol_wid + 1) + '3')
657
658
659 iRow1 = 0
660 iRow2 = 1
661 iCol = -1
662 for header, subheaders in get_excel_headers(model, is_user_anonymous=is_user_anonymous):
663 if len(subheaders) <= 1:
664 iCol += 1
665 cell = ws.cell(row=iRow2, column=iCol)
666 cell.set_value_explicit(value = header, data_type = Cell.TYPE_STRING)
667 else:
668 ws.merge_cells(start_row=iRow1, start_column=iCol+1, end_row=iRow1, end_column=iCol+len(subheaders))
669 cell = ws.cell(row=iRow1, column=iCol+1)
670 cell.set_value_explicit(value = header, data_type = Cell.TYPE_STRING)
671 for subheader in subheaders:
672 iCol += 1
673 cell = ws.cell(row=iRow2, column=iCol)
674 cell.set_value_explicit(value = subheader, data_type = Cell.TYPE_STRING)
675
676
677 iRow = 1
678 for obj in objectDict[model_type]:
679 iRow += 1
680 iCol = -1
681 for field in fields:
682 value, data_type, format_code = format_value_excel(obj, field)
683 try:
684 delattr(obj, "_" + field.name + "_cache")
685 except Exception, e:
686 pass
687
688 if not isinstance(value, list):
689 value = [value,]
690 data_type = [data_type, ]
691 format_code = [format_code, ]
692
693 for iter in range(len(value)):
694 iCol += 1
695 cell = ws.cell(row=iRow, column=iCol)
696 if value[iter] is None:
697 cell.set_value_explicit(b'', data_type = Cell.TYPE_NULL)
698 else:
699 cell.set_value_explicit(value = value[iter], data_type = data_type[iter])
700 cell.style.number_format.format_code = format_code[iter]
701 cell.style.font.name = 'Arial'
702 cell.style.font.size = 10
703 cell.style.alignment.wrap_text = True
704 cell.style.fill.fill_type = 'solid'
705 if iRow % 2 == 0:
706 cell.style.fill.start_color.index = Colors.GREY1
707 else:
708 cell.style.fill.start_color.index = Colors.GREY2
709
710
711 nCols = ws.get_highest_column()
712 nRows = ws.get_highest_row()
713 for iRow in range(nRows):
714 cell = ws.cell(row=iRow, column=0)
715 cell.style.font.bold = True
716 if iRow > 1:
717 if iRow % 2 == 0:
718 cell.style.fill.start_color.index = Colors.GREY3
719 else:
720 cell.style.fill.start_color.index = Colors.GREY4
721 if iRow > 0:
722 cell.style.borders.left.color.index = Colors.BLUE
723 cell.style.borders.left.border_style = Border.BORDER_THIN
724 cell.style.borders.right.color.index = Colors.BLUE
725 cell.style.borders.right.border_style = Border.BORDER_THIN
726
727 cell = ws.cell(row=iRow, column=nCols-1)
728 cell.style.borders.right.color.index = Colors.BLUE
729 cell.style.borders.right.border_style = Border.BORDER_THIN
730
731 ws.row_dimensions[iRow + 1].height = 15
732 for iCol in range(nCols):
733 cell = ws.cell(row=0, column=iCol)
734 if cell.value is not None:
735 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT
736 cell.style.font.name = 'Arial'
737 cell.style.font.size = 10
738 cell.style.font.color.index = Colors.WHITE
739 cell.style.font.bold = True
740 cell.style.alignment.horizontal = Alignment.HORIZONTAL_CENTER
741 cell.style.alignment.wrap_text = True
742 cell.style.fill.fill_type = 'solid'
743 cell.style.fill.start_color.index = Colors.BLUE
744 cell.style.borders.top.color.index = Colors.BLUE
745 cell.style.borders.top.border_style = Border.BORDER_THIN
746 if (iCol == 0) or (ws.cell(row=0, column=iCol-1).value is None):
747 cell.style.borders.left.color.index = Colors.BLUE
748 cell.style.borders.left.border_style = Border.BORDER_THIN
749 if (iCol == nCols - 1) or (ws.cell(row=0, column=iCol+1).value is None):
750 cell.style.borders.right.color.index = Colors.BLUE
751 cell.style.borders.right.border_style = Border.BORDER_THIN
752 else:
753 cell.style.borders.bottom.color.index = Colors.BLUE
754 cell.style.borders.bottom.border_style = Border.BORDER_THIN
755
756 cell = ws.cell(row=1, column=iCol)
757 cell.style.number_format.format_code = NumberFormat.FORMAT_TEXT
758 cell.style.font.name = 'Arial'
759 cell.style.font.size = 10
760 cell.style.font.color.index = Colors.WHITE
761 cell.style.font.bold = True
762 cell.style.alignment.wrap_text = True
763 cell.style.fill.fill_type = 'solid'
764 cell.style.fill.start_color.index = Colors.BLUE
765 cell.style.borders.bottom.color.index = Colors.BLUE
766 cell.style.borders.bottom.border_style = Border.BORDER_THIN
767
768 cell = ws.cell(row=nRows-1, column=iCol)
769 cell.style.borders.bottom.color.index = Colors.BLUE
770 cell.style.borders.bottom.border_style = Border.BORDER_THIN
771
772 return wb
773
775 fields = getModelDataFields(model, metadata=not is_user_anonymous)
776 headers = []
777 for field in fields:
778 subheaders = []
779 if (field.rel is not None) and isinstance(field, ForeignKey) and (not issubclass(field.rel.to, (Entry, User, ))):
780 for subfield in field.rel.to._meta.fields + field.rel.to._meta.many_to_many:
781 if not subfield.auto_created:
782 subheaders.append(html_to_ascii(subfield.verbose_name))
783 headers.append((html_to_ascii(field.verbose_name), subheaders, ))
784 return headers
785
880
882 models = get_models_in_saving_order()
883
884
885 data = read_excel_data(fileName)
886
887
888 if data.has_key('Species'):
889 if len(data['Species']) > 1:
890 raise ValidationError('Please define species 1 at a time.')
891 species = data['Species'][0]
892 if species.has_key('id'):
893 species_id = species['id']
894 else:
895 species_id = None
896 species_wid = species['wid']
897 else:
898 try:
899 species = Species.objects.get(wid=species_wid)
900 species_id = species.id
901 except ObjectDoesNotExist:
902 raise ValidationError('Please edit an editing PGDB')
903
904 if len(Species.objects.exclude(id=species_id).filter(wid=species_wid)) > 0:
905 raise ValidationError('Species WID must be unique.')
906
907
908 old_wids_list = []
909 if species_id is not None:
910 qs = SpeciesComponent.objects.values('wid', 'model_type').filter(species__id=species_id)
911 for model in models:
912 if data.has_key(model.__name__):
913 for obj_data in data[model.__name__]:
914 if obj_data['id'] is not None:
915 qs = qs.exclude(id=obj_data['id'])
916
917 for x in qs:
918 old_wids_list.append(x['wid'])
919
920 new_wids_list = []
921 for model in models:
922 if data.has_key(model.__name__):
923 for obj_data in data[model.__name__]:
924 new_wids_list.append(obj_data['wid'])
925 else:
926 data[model.__name__] = []
927
928 if len(set(new_wids_list)) < len(new_wids_list):
929 print new_wids_list
930 raise ValidationError('WIDs must be unique')
931 if len(set(new_wids_list) & set(old_wids_list)) > 0:
932 raise ValidationError('WIDs not unique: %s' % ', '.join(set(new_wids_list) & set(old_wids_list)))
933
934
935 all_obj_wids = {species_wid: 'Species'}
936 all_obj_data = {species_wid: species}
937 all_obj_data_by_model = {'Species': [species]}
938 for model_name, model in getModels().iteritems():
939 if not issubclass(model, SpeciesComponent):
940 continue
941 all_obj_data_by_model[model_name] = []
942 for obj in data[model_name]:
943 all_obj_wids[obj['wid']] = obj['model_type']
944 all_obj_data[obj['wid']] = obj
945 all_obj_data_by_model[model_name].append(obj)
946
947 if species_id is not None:
948 qs = model.objects.select_related().filter(species__id=species_id)
949 for obj in data[model_name]:
950 if obj_data['id'] is not None:
951 qs = qs.exclude(id=obj['id'])
952 for obj in qs:
953 all_obj_wids[obj.wid] = obj.model_type
954 all_obj_data[obj.wid] = obj
955 all_obj_data_by_model[model_name].append(obj)
956
957 ''' validation '''
958 errors = []
959
960
961 for model in models:
962 for obj_data in data[model.__name__]:
963 try:
964 obj_data = validate_object_fields(model, obj_data, all_obj_wids, species_wid, obj_data['wid'])
965 except ValidationError as error:
966 errors.append('%s %s invalid: %s' % (model.__name__, obj_data['wid'], format_error_as_html_list(error)))
967 if len(errors) > 0:
968 raise ValidationError(format_list_html(errors))
969
970
971 errors = []
972 for model in models:
973 for obj_data in data[model.__name__]:
974 try:
975 validate_model_objects(model,
976 obj_data,
977 all_obj_data=all_obj_data,
978 all_obj_data_by_model=all_obj_data_by_model)
979 except ValidationError as error:
980 errors.append('%s %s invalid: %s' % (model.__name__, obj_data['wid'], format_error_as_html_list(error)))
981 if len(errors) > 0:
982 raise ValidationError(format_list_html(errors))
983
984
985 for model in models:
986 try:
987 validate_model_unique(model,
988 all_obj_data_by_model[model.__name__],
989 all_obj_data=all_obj_data,
990 all_obj_data_by_model=all_obj_data_by_model)
991 except ValidationError as error:
992 errors.append('%s invalid: %s' % (model.__name__, format_error_as_html_list(error)))
993 if len(errors) > 0:
994 raise ValidationError(format_list_html(errors))
995
996 ''' save '''
997
998 obj_list = {}
999 for model in models:
1000 for obj_data in data[model.__name__]:
1001 if obj_data['id'] is None:
1002 obj = model()
1003 else:
1004 obj = model.objects.get(id = obj_data['id'])
1005 obj_list[obj_data['wid']] = save_object_data(species_wid, obj, obj_data, obj_list, user, save=False, save_m2m=False)
1006
1007
1008 for model in models:
1009 for obj_data in data[model.__name__]:
1010 obj = obj_list[obj_data['wid']]
1011 obj_list[obj_data['wid']] = save_object_data(species_wid, obj, obj_data, obj_list, user, save=True, save_m2m=False)
1012
1013
1014 for model in models:
1015 for obj_data in data[model.__name__]:
1016 obj = obj_list[obj_data['wid']]
1017 obj_list[obj_data['wid']] = save_object_data(species_wid, obj, obj_data, obj_list, user, save=True, save_m2m=True)
1018
1020
1021 wb = load_workbook(filename = filename)
1022
1023
1024 models = getModels()
1025
1026
1027 errors = []
1028
1029
1030 for modelname, model in models.iteritems():
1031 ws = wb.get_sheet_by_name(name = model._meta.verbose_name_plural[:31])
1032 if ws is None:
1033 continue
1034
1035 iRow1 = 0
1036 iRow2 = 1
1037 iCol = -1
1038 for header, subheaders in get_excel_headers(model):
1039 if len(subheaders) <= 1:
1040 iCol += 1
1041 cell = ws.cell(row=iRow2, column=iCol)
1042 if cell.value != header:
1043 errors.append('Expecting worksheet "%s" header %s%s "%s", got "%s"' % (ws.title, get_column_letter(iCol+1), iRow2+1, header, cell.value))
1044 else:
1045 cell = ws.cell(row=iRow1, column=iCol+1)
1046 if cell.value != header:
1047 errors.append('Expecting worksheet "%s" header %s%s "%s", got "%s"' % (ws.title, get_column_letter(iCol+2), iRow1+1, header, cell.value))
1048 for subheader in subheaders:
1049 iCol += 1
1050 cell = ws.cell(row=iRow2, column=iCol)
1051 if cell.value != subheader:
1052 errors.append('Expecting worksheet "%s" header %s%s "%s", got "%s"' % (ws.title, get_column_letter(iCol+1), iRow2+1, subheader, cell.value))
1053
1054 if (iCol + 1) != ws.get_highest_column():
1055 errors.append('Expecting worksheet "%s" to have exactly %s columns' % (ws.title, iCol + 1))
1056
1057 if len(errors) > 0:
1058 raise ValidationError(format_list_html(errors))
1059
1060
1061 newobjects = {}
1062 for modelname, model in models.iteritems():
1063 ws = wb.get_sheet_by_name(name = model._meta.verbose_name_plural[:31])
1064 if ws is None:
1065 continue
1066
1067 fields = getModelDataFields(model)
1068
1069 iCol = -1
1070 iCol_id = None
1071 for header, subheaders in get_excel_headers(model):
1072 if header == 'ID':
1073 iCol_id = iCol + 1
1074 iCol += max(1, len(subheaders))
1075
1076
1077 newobjects[modelname] = []
1078 for iRow in range(2, ws.get_highest_row()):
1079 id = ws.cell(row=iRow, column=iCol_id).value
1080 if id == '':
1081 id is None
1082 obj = {'id': id, 'model_type': modelname}
1083
1084 iField = -1
1085 iCol = -1
1086 for header, subheaders in get_excel_headers(model):
1087 iField += 1
1088 iCol += max(1, len(subheaders))
1089 field = fields[iField]
1090
1091 if not field.editable or field.auto_created:
1092 continue
1093
1094 if isinstance(field, (CharField, SlugField, TextField)):
1095 value = ws.cell(row=iRow, column=iCol).value
1096 if value is None:
1097 value = ''
1098 obj[field.name] = value
1099 elif isinstance(field, (BooleanField, FloatField, IntegerField, NullBooleanField, PositiveIntegerField)):
1100 value = ws.cell(row=iRow, column=iCol).value
1101 if value == '':
1102 value = None
1103 obj[field.name] = value
1104 elif isinstance(field, ForeignKey):
1105 if issubclass(field.rel.to, Entry):
1106 value = ws.cell(row=iRow, column=iCol).value
1107 if value == '':
1108 value = None
1109 obj[field.name] = value
1110 else:
1111 obj[field.name] = {}
1112 iCol2 = iCol - max(1, len(subheaders))
1113 is_blank = True
1114 for subfield in field.rel.to._meta.fields + field.rel.to._meta.many_to_many:
1115 if subfield.auto_created:
1116 continue
1117
1118 iCol2 += 1
1119 if isinstance(subfield, (CharField, SlugField, TextField)):
1120 value = ws.cell(row=iRow, column=iCol2).value
1121 if value is None:
1122 value = ''
1123 elif isinstance(subfield, (BooleanField, FloatField, IntegerField, NullBooleanField, PositiveIntegerField)):
1124 value = ws.cell(row=iRow, column=iCol2).value
1125 if value == '':
1126 value = None
1127 elif isinstance(subfield, ForeignKey) and issubclass(subfield.rel.to, Entry):
1128 value = ws.cell(row=iRow, column=iCol2).value
1129 if value == '':
1130 value = None
1131 elif isinstance(subfield, ManyToManyField) and issubclass(subfield.rel.to, Entry):
1132 value = ws.cell(row=iRow, column=iCol2).value
1133 if value is None or value == '':
1134 value = []
1135 else:
1136 value = value.split(', ')
1137 elif isinstance(subfield, ForeignKey):
1138 value = ws.cell(row=iRow, column=iCol2).value
1139 if value == '':
1140 value = None
1141 else:
1142 value = simplejson.loads(value)
1143 elif isinstance(subfield, ManyToManyField):
1144 value = ws.cell(row=iRow, column=iCol2).value
1145 if value is None or value == '':
1146 value = []
1147 else:
1148 value = simplejson.loads('[' + value + ']')
1149
1150 obj[field.name][subfield.name] = value
1151
1152 if value is not None and not (isinstance(value, (str, unicode)) and value == '') and not (isinstance(value, list) and len(value) == 0):
1153 is_blank = False
1154
1155 if is_blank:
1156 obj[field.name] = None
1157 elif isinstance(field, ManyToManyField):
1158 if issubclass(field.rel.to, Entry):
1159 value = ws.cell(row=iRow, column=iCol).value
1160 if value is None or value == '':
1161 obj[field.name] = []
1162 else:
1163 obj[field.name] = value.split(', ')
1164 else:
1165 value = ws.cell(row=iRow, column=iCol).value
1166 if value is None or value == '':
1167 obj[field.name] = []
1168 else:
1169 try:
1170 obj[field.name] = simplejson.loads('[' + value + ']')
1171 except:
1172 errors.append('Invalid json at field %s of %s %s at cell %s%s' % (field.name, modelname, obj['wid'], get_column_letter(iCol+1), iRow+1, ))
1173
1174 newobjects[modelname].append(obj)
1175
1176 if len(errors) > 0:
1177 raise ValidationError(format_list_html(errors))
1178
1179 return newobjects
1180
1193
1195 for model in get_models_in_saving_order():
1196 for obj in newobjects[model.__name__]:
1197 try:
1198 obj.delete()
1199 except:
1200 pass
1201 for obj in newsubobjects:
1202 if obj.pk is not None:
1203 try:
1204 obj.delete()
1205 except:
1206 pass
1207
1209 if model is None:
1210 model = obj.__class__
1211 qs = model.objects.none()
1212 qs._result_cache.append(obj)
1213 return qs
1214
1318
1321
1323 model = obj.__class__
1324
1325 if issubclass(model, Entry) and len(ancestors) > 0:
1326 return obj.wid
1327
1328 objDict = {}
1329 if issubclass(model, Entry):
1330 fields = getModelDataFields(model, metadata=not is_user_anonymous)
1331 else:
1332 fields = model._meta.fields + model._meta.many_to_many
1333
1334 for field in fields:
1335 model_val = getattr(obj, field.name)
1336 if field.auto_created:
1337 continue
1338
1339 if isinstance(field, ForeignKey):
1340 if model_val is None:
1341 stdobj_val = None
1342 else:
1343 if issubclass(field.rel.to, Entry):
1344 stdobj_val = model_val.wid
1345 elif issubclass(field.rel.to, User):
1346 stdobj_val = model_val.username
1347 else:
1348 stdobj_val = convert_modelobject_to_stdobject(model_val, is_user_anonymous, ancestors + [obj])
1349 elif isinstance(field, ManyToManyField):
1350 stdobj_val = []
1351 if issubclass(field.rel.to, Entry):
1352 for model_subval in model_val.all():
1353 stdobj_val.append(model_subval.wid)
1354 elif issubclass(field.rel.to, User):
1355 for model_subval in model_val.all():
1356 stdobj_val.append(model_subval.username)
1357 else:
1358 for model_subval in model_val.all():
1359 stdobj_subval = convert_modelobject_to_stdobject(model_subval, is_user_anonymous, ancestors + [obj])
1360 stdobj_val.append(stdobj_subval)
1361 elif field.choices is not None and len(field.choices) > 0:
1362 choices = [x[0] for x in field.choices]
1363 if model_val in choices:
1364 stdobj_val = unicode(field.choices[choices.index(model_val)][1])
1365 else:
1366 stdobj_val = unicode(model_val)
1367 else:
1368 stdobj_val = unicode(model_val)
1369
1370 objDict[field.name] = stdobj_val
1371
1372 for field in model._meta.get_all_related_objects() + model._meta.get_all_related_many_to_many_objects():
1373 if field.get_accessor_name() == '+':
1374 continue
1375
1376 objDict[field.get_accessor_name()] = []
1377 for model_subval in getattr(obj, field.get_accessor_name()).all():
1378 if model_subval not in ancestors:
1379 objDict[field.get_accessor_name()].append(convert_modelobject_to_stdobject(model_subval, is_user_anonymous, ancestors + [obj]))
1380
1381 return objDict
1382
1384 model = obj.__class__
1385 xml_obj = xml_doc.createElement('object')
1386
1387 if issubclass(model, Entry) and len(ancestors) > 0:
1388 xml_obj.appendChild(xml_doc.createTextNode(unicode(obj.wid)))
1389 return xml_obj
1390
1391 xml_field = xml_doc.createElement('field')
1392 xml_field.setAttribute('name', 'model')
1393 xml_field.appendChild(xml_doc.createTextNode(unicode(model.__name__)))
1394 xml_obj.appendChild(xml_field)
1395
1396 if issubclass(model, Entry):
1397 fields = getModelDataFields(model, metadata=not is_user_anonymous)
1398 else:
1399 fields = model._meta.fields + model._meta.many_to_many
1400
1401 for field in fields:
1402 model_val = getattr(obj, field.name)
1403 if field.auto_created:
1404 continue
1405
1406 xml_field = xml_doc.createElement('field')
1407 xml_field.setAttribute('name', field.name)
1408 xml_obj.appendChild(xml_field)
1409
1410 if isinstance(field, ForeignKey):
1411 if model_val is None:
1412 pass
1413 else:
1414 if issubclass(field.rel.to, Entry):
1415 xml_field.appendChild(xml_doc.createTextNode(unicode(model_val.wid)))
1416 elif issubclass(field.rel.to, User):
1417 xml_field.appendChild(xml_doc.createTextNode(unicode(model_val.username)))
1418 else:
1419 xml_field.appendChild(convert_modelobject_to_xml(model_val, xml_doc, is_user_anonymous, ancestors + [obj]))
1420 elif isinstance(field, ManyToManyField):
1421 if issubclass(field.rel.to, Entry):
1422 for model_subval in model_val.all():
1423 doc = xml_doc.createElement('object')
1424 doc.appendChild(xml_doc.createTextNode(unicode(model_subval.wid)))
1425 xml_field.appendChild(doc)
1426 elif issubclass(field.rel.to, User):
1427 for model_subval in model_val.all():
1428 doc = xml_doc.createElement('object')
1429 doc.appendChild(xml_doc.createTextNode(unicode(model_subval.username)))
1430 xml_field.appendChild(doc)
1431 else:
1432 for model_subval in model_val.all():
1433 xml_field.appendChild(convert_modelobject_to_xml(model_subval, xml_doc, is_user_anonymous, ancestors + [obj]))
1434 elif field.choices is not None and len(field.choices) > 0:
1435 choices = [x[0] for x in field.choices]
1436 if model_val in choices:
1437 xml_field.appendChild(xml_doc.createTextNode(unicode(field.choices[choices.index(model_val)][1])))
1438 else:
1439 xml_field.appendChild(xml_doc.createTextNode(unicode(model_val)))
1440 else:
1441 xml_field.appendChild(xml_doc.createTextNode(unicode(model_val)))
1442
1443 for field in model._meta.get_all_related_objects() + model._meta.get_all_related_many_to_many_objects():
1444 if field.get_accessor_name() == '+':
1445 continue
1446
1447 xml_field = xml_doc.createElement('field')
1448 xml_field.setAttribute('name', field.get_accessor_name())
1449 xml_obj.appendChild(xml_field)
1450
1451 for model_subval in getattr(obj, field.get_accessor_name()).all():
1452 if model_subval not in ancestors:
1453 xml_field.appendChild(convert_modelobject_to_xml(model_subval, xml_doc, is_user_anonymous, ancestors + [obj]))
1454
1455 return xml_obj
1456
1545
1600
1602 if issubclass(model, Entry):
1603 fields = [model._meta.get_field_by_name(x)[0] for x in model._meta.field_list]
1604 else:
1605 fields = model._meta.fields + model._meta.many_to_many
1606
1607 if issubclass(model, Entry):
1608 data['species'] = species_wid
1609
1610 errors = {}
1611 for field in fields:
1612 if not field.editable or field.auto_created:
1613 continue
1614
1615 try:
1616 if isinstance(field, ForeignKey):
1617 if issubclass(model, Evidence) and issubclass(field.rel.to, SpeciesComponent):
1618 data[field.name] = entry_wid
1619
1620 if not data.has_key(field.name):
1621 data[field.name] = None
1622
1623 if data[field.name] is None:
1624 if not field.null:
1625 raise ValidationError('Undefined WID %s' % data[field.name])
1626 else:
1627 if issubclass(field.rel.to, Entry):
1628 if not wids.has_key(data[field.name]):
1629 raise ValidationError('Undefined WID %s' % data[field.name])
1630 if not issubclass(getModel(wids[data[field.name]]), field.rel.to):
1631 raise ValidationError('Invalid WID %s' % data[field.name])
1632 else:
1633 try:
1634 data[field.name] = validate_object_fields(field.rel.to, data[field.name], wids, species_wid, entry_wid)
1635 except ValidationError as error:
1636 tmp = ['%s: %s' % (key, val, ) for key, val in error.message_dict.iteritems()]
1637 raise ValidationError('<ul><li>%s</li></ul>' % '</li><li>'.join(tmp))
1638
1639 elif isinstance(field, ManyToManyField):
1640 if not data.has_key(field.name):
1641 data[field.name] = []
1642
1643 if issubclass(field.rel.to, Entry):
1644 for wid in data[field.name]:
1645 if not wids.has_key(wid):
1646 raise ValidationError('Undefined WID %s' % wid)
1647 if not issubclass(getModel(wids[wid]), field.rel.to):
1648 raise ValidationError('Invalid WID %s' % wid)
1649 else:
1650 for idx in range(len(data[field.name])):
1651 try:
1652 data[field.name][idx] = validate_object_fields(field.rel.to, data[field.name][idx], wids, species_wid, entry_wid)
1653 except ValidationError as error:
1654 tmp = ['%s: %s' % (key, val, ) for key, val in error.message_dict.iteritems()]
1655 raise ValidationError('<ul><li>%s</li></ul>' % '</li><li>'.join(tmp))
1656 else:
1657 if data.has_key(field.name):
1658 data[field.name] = field.clean(data[field.name], None)
1659 else:
1660 if field.default != NOT_PROVIDED:
1661 data[field.name] = field.clean(field.default, None)
1662 else:
1663 data[field.name] = field.clean(None, None)
1664
1665 except ValidationError as error:
1666 errors[field.name] = ', '.join(error.messages)
1667
1668 if len(errors) > 0:
1669 raise ValidationError(errors)
1670
1671 return data
1672
1674 parent_list = list(model._meta.get_parent_list())
1675 parent_list.reverse()
1676 for parent_model in parent_list + [model]:
1677 if hasattr(parent_model._meta, 'clean'):
1678 parent_model._meta.clean(parent_model, obj_data, all_obj_data=all_obj_data, all_obj_data_by_model=all_obj_data_by_model)
1679
1681 if not issubclass(model, SpeciesComponent):
1682 return
1683
1684 parent_list = list(model._meta.get_parent_list())
1685 parent_list.reverse()
1686
1687 if all_obj_data is None:
1688 if model_objects_data[0] is Entry:
1689 species_wid = model_objects_data[0].species.wid
1690 else:
1691 species_wid = model_objects_data[0]['species']
1692
1693 if issubclass(model, SpeciesComponent):
1694 species_id = Species.objects.values('id').get(wid=species_wid)['id']
1695 qs = model.objects.filter(species__id=species_id)
1696 else:
1697 qs = model.objects.all()
1698 for obj in model_objects_data:
1699 if obj['id'] is not None:
1700 qs = qs.exclude(id=obj['id'])
1701 for obj in qs:
1702 model_objects_data.append(obj)
1703
1704 for ancestor_model in parent_list + [model]:
1705 if hasattr(ancestor_model._meta, 'validate_unique'):
1706 ancestor_model._meta.validate_unique(ancestor_model, model_objects_data,
1707 all_obj_data=all_obj_data, all_obj_data_by_model=all_obj_data_by_model)
1708
1709 -def save_object_data(species_wid, obj, obj_data, obj_list, user, save=False, save_m2m=False):
1710 model = obj.__class__
1711 if isinstance(obj, Entry) and obj.pk is None:
1712 obj.created_user = user
1713 obj.last_updated_user = user
1714
1715 if issubclass(model, Entry):
1716 fields = [model._meta.get_field_by_name(x)[0] for x in model._meta.field_list]
1717 else:
1718 fields = model._meta.fields + model._meta.many_to_many
1719
1720
1721 for field in fields:
1722 if not field.editable or field.auto_created:
1723 continue
1724
1725 if isinstance(field, ForeignKey):
1726 if save:
1727 if issubclass(field.rel.to, Entry):
1728 if obj_data[field.name] is not None:
1729 if obj_list.has_key(obj_data[field.name]):
1730 tmp = obj_list[obj_data[field.name]]
1731 elif issubclass(field.rel.to, SpeciesComponent):
1732 tmp = field.rel.to.objects.get(species__wid=species_wid, wid=obj_data[field.name])
1733 else:
1734 tmp = field.rel.to.objects.get(wid=obj_data[field.name])
1735 setattr(obj, field.name, tmp)
1736 else:
1737 setattr(obj, field.name, None)
1738 else:
1739 if obj_data[field.name] is not None:
1740 setattr(obj, field.name, save_object_data(species_wid, field.rel.to(), obj_data[field.name], obj_list, user, save=save, save_m2m=save_m2m))
1741 else:
1742 setattr(obj, field.name, None)
1743 elif isinstance(field, ManyToManyField):
1744 pass
1745 else:
1746 setattr(obj, field.name, obj_data[field.name])
1747
1748
1749 if save:
1750 if isinstance(obj, SpeciesComponent):
1751 obj.species = Species.objects.get(wid=species_wid)
1752 obj.full_clean()
1753 obj.save()
1754
1755
1756 for field in fields:
1757 if not field.editable or field.auto_created:
1758 continue
1759
1760 if isinstance(field, ManyToManyField):
1761 if save_m2m:
1762 getattr(obj, field.name).clear()
1763 if issubclass(field.rel.to, Entry):
1764 for wid in obj_data[field.name]:
1765 if obj_list.has_key(wid):
1766 tmp = obj_list[wid]
1767 elif issubclass(field.rel.to, SpeciesComponent):
1768 tmp = field.rel.to.objects.get(species__wid=species_wid, wid=wid)
1769 else:
1770 tmp = field.rel.to.objects.get(wid=wid)
1771 getattr(obj, field.name).add(tmp)
1772 else:
1773 for sub_obj_data in obj_data[field.name]:
1774 getattr(obj, field.name).add(save_object_data(species_wid, field.rel.to(), sub_obj_data, obj_list, user, save=save, save_m2m=save_m2m))
1775
1776
1777 if save:
1778 obj.full_clean()
1779 obj.save()
1780
1781
1782 return obj
1783
1795
1797 error_messages = []
1798
1799 f = open(filename, 'r')
1800 data = f.readlines()
1801 f.close()
1802
1803 wid = None
1804 sequences = {}
1805 for i in range(len(data)):
1806 if data[i][0] == '>':
1807 wid = data[i].strip()[1:]
1808 sequences[wid] = ''
1809 else:
1810 if wid is None:
1811 return (False, 'File does not match FASTA format.')
1812 sequences[wid] += data[i].strip()
1813
1814
1815 for wid, sequence in sequences.iteritems():
1816 try:
1817 chr = Chromosome.objects.get(species__wid=species_wid, wid=wid)
1818 except ObjectDoesNotExist as error:
1819 error_messages.append(error.message)
1820 continue
1821 if len(error_messages) > 0:
1822 return (False, format_list_html(error_messages))
1823
1824
1825 for wid, sequence in sequences.iteritems():
1826 chr = Chromosome.objects.get(species__wid=species_wid, wid=wid)
1827 chr.sequence = sequence
1828 try:
1829 chr.full_clean()
1830 except ValidationError as error:
1831 error_messages.append(format_error_as_html_list(error))
1832 continue
1833 if len(error_messages) > 0:
1834 return (False, format_list_html(error_messages))
1835
1836
1837
1838 for wid, sequence in sequences.iteritems():
1839 chr = Chromosome.objects.get(species__wid=species_wid, wid=wid)
1840 chr.sequence = sequence
1841 chr.full_clean()
1842 try:
1843 chr.save()
1844 except ValueError as error:
1845 error_messages.append(error.message)
1846 continue
1847 if len(error_messages) > 0:
1848 return (False, format_list_html(error_messages))
1849
1850 return (True, 'Sequences successfully saved!')
1851
1853 refs = {}
1854 for obj in qs:
1855 if not isinstance(obj, SpeciesComponent):
1856 continue
1857
1858 if isinstance(obj, Reference):
1859 refs[obj.wid] = obj
1860 for sub_obj in obj.references.all():
1861 refs[sub_obj.wid] = sub_obj
1862 for field in obj.__class__._meta.fields + obj.__class__._meta.many_to_many:
1863 if isinstance(field, ForeignKey) and issubclass(field.rel.to, EvidencedEntryData) and getattr(obj, field.name) is not None:
1864 for evidence in getattr(obj, field.name).evidence.all():
1865 for ref in evidence.references.all():
1866 refs[ref.wid] = ref
1867 if isinstance(field, ManyToManyField) and issubclass(field.rel.to, EvidencedEntryData):
1868 for sub_obj in getattr(obj, field.name).all():
1869 for evidence in sub_obj.evidence.all():
1870 for ref in evidence.references.all():
1871 refs[ref.wid] = ref
1872
1873 bibs = []
1874 wids = refs.keys()
1875 wids.sort()
1876 for wid in wids:
1877 bibs.append(refs[wid].get_as_bibtex())
1878
1879 now = datetime.datetime.now(tzlocal())
1880 return \
1881 '@preamble {"\n' \
1882 + ('\t%s WholeCellKB Bibliography\n' % (species.name, )) \
1883 + ('\tGenerated by WholeCellKB on %s at %s\n' % (
1884 now.isoformat(),
1885 settings.ROOT_URL + reverse('public.views.exportData', kwargs={'species_wid': species.wid}),
1886 )) \
1887 + ('\t%s %s %s\n' % (html_to_ascii('©'), now.year, 'Covert Lab, Department of Bioengineering, Stanford University', )) \
1888 + '"}\n\n' \
1889 + '\n\n'.join(bibs)
1890
1891 -def get_invalid_objects(species_id, validate_fields=False, validate_objects=True, full_clean=False, validate_unique=False):
1892
1893 wids = []
1894 for species in Species.objects.values('wid').all():
1895 wids.append(species['wid'])
1896 if len(set(wids)) < len(wids):
1897 for wid in set(wids):
1898 del wids[wids.index(wid)]
1899 return [{
1900 'type': 'model',
1901 'name': model_name,
1902 'url': None,
1903 'message': 'The following species WIDs are not unique:<ul><li>%s</li></ul>' % '</li></li>'.join(wids),
1904 }]
1905
1906
1907 all_obj_wids = {}
1908 all_obj_data = {}
1909 all_obj_data_by_model = {}
1910
1911 species = Species.objects.select_related().get(id=species_id)
1912 species_wid = species.wid
1913 all_obj_wids[species.wid] = 'Species'
1914 all_obj_data[species.wid] = species
1915 all_obj_data_by_model['Species'] = [species]
1916
1917 wids = []
1918 for model_name, model in getModels(SpeciesComponent).iteritems():
1919 all_obj_data_by_model[model_name] = []
1920 for obj in model.objects.select_related().filter(species__id=species_id):
1921 wids.append(obj.wid)
1922 all_obj_wids[obj.wid] = model_name
1923 all_obj_data[obj.wid] = obj
1924 all_obj_data_by_model[model_name].append(obj)
1925
1926
1927 if len(set(wids)) < len(wids):
1928 for wid in set(wids):
1929 del wids[wids.index(wid)]
1930 return [{
1931 'type': 'model',
1932 'name': model_name,
1933 'url': None,
1934 'message': 'The following WIDs are not unique:<ul><li>%s</li></ul>' % '</li></li>'.join(wids),
1935 }]
1936
1937
1938 errors = []
1939 for model_name, model in getModels().iteritems():
1940 for obj in all_obj_data_by_model[model_name]:
1941 junk, obj_data = get_edit_form_fields(species_wid, model, obj=obj)
1942 obj_data['species'] = species_wid
19