############################################################################## # # Photo for Zope # # Copyright (c) 2004 Soren Roug, All rights reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # ############################################################################## __doc__ = """ ExtFile product module. The ExtFile-Product works like the Zope File-product, but stores the uploaded file externally in a repository-direcory. $Id: ExtFile.py 5740 2006-01-22 11:38:09Z roug $ """ __version__='$Release: 1.1.2 $'[10:-2] import Globals from __main__ import * #import Zope from AccessControl import getSecurityManager, ClassSecurityInfo from OFS.SimpleItem import SimpleItem from OFS.PropertyManager import PropertyManager from OFS.ObjectManager import BeforeDeleteException from Globals import DTMLFile, MessageDialog from zope.app.content_types import guess_content_type from webdav.common import rfc1123_date from DateTime import DateTime from webdav.WriteLockInterface import WriteLockInterface from mimetools import choose_boundary from ZPublisher import HTTPRangeSupport import urllib, os, types, string from os.path import join, isfile import stat try: from cStringIO import StringIO except: from StringIO import StringIO bad_chars = ' ,+&;()[]{}\xC4\xC5\xC1\xC0\xC2\xC3' \ '\xE4\xE5\xE1\xE0\xE2\xE3\xC7\xE7\xC9\xC8\xCA\xCB' \ '\xC6\xE9\xE8\xEA\xEB\xE6\xCD\xCC\xCE\xCF\xED\xEC' \ '\xEE\xEF\xD1\xF1\xD6\xD3\xD2\xD4\xD5\xD8\xF6\xF3' \ '\xF2\xF4\xF5\xF8\x8A\x9A\xDF\xDC\xDA\xD9\xDB\xFC' \ '\xFA\xF9\xFB\xDD\x9F\xFD\xFF\x8E\x9E' good_chars= '___________AAAAAA' \ 'aaaaaaCcEEEE' \ 'EeeeeeIIIIii' \ 'iiNnOOOOOOoo' \ 'ooooSssUUUUu' \ 'uuuYYyyZz' TRANSMAP = string.maketrans(bad_chars, good_chars) def cleanup_id(str): """ Cleanup an id Should be more thorough and e.g. remove trailing dots/spaces """ return string.translate(str, TRANSMAP) FLAT = 0 SYNC_ZODB = 1 SLICED = 2 REPOSITORY = SYNC_ZODB # format for the files in the repository: # %u=user, %p=path, %n=file name, %e=file extension, %c=counter, %t=time FILE_FORMAT = "%n%c%e" BACKUP_ON_DELETE = 0 ALWAYS_BACKUP = 1 UNDO_POLICY = BACKUP_ON_DELETE def manage_addExtFile(self, id='', title='', file='', content_type='', REQUEST=None): """Add a ExtFile to a folder.""" # content_type argument is probably not used anywhere. if id=='' and type(file) is not type('') and hasattr(file,'filename'): # generate id from filename and make sure, there are no spaces in the id id=file.filename id=id[max(string.rfind(id,'/'), string.rfind(id,'\\'), string.rfind(id,':') )+1:] id = cleanup_id(id) if id: self=self.this() self._setObject(id, ExtFile(id, title)) self._getOb(id).manage_file_upload(file, content_type) if REQUEST is not None: security=getSecurityManager() if security.checkPermission('View management screens',self): ppath = './manage_main' else: pobj = REQUEST.PARENTS[0] ppath = string.join(pobj.getPhysicalPath(), '/') return MessageDialog(title = 'Created', message = 'The ExtFile %s was successfully created!' % id, action = ppath,) else: if REQUEST is not None: return MessageDialog(title = 'Error!', message = 'You must specify a file!', action = './manage_main',) class ExtFile(SimpleItem, PropertyManager): "An External ExtFile allows indexing and conversions." __implements__ = (WriteLockInterface,) # what properties have we? _properties=( {'id':'title', 'type':'string'}, {'id':'descr', 'type':'string'}, {'id':'content_type', 'type':'string'}, ) # what management options are there? manage_options = ( {'label':'Edit', 'action': 'manage_main' }, {'label':'View/Download', 'action': '' }, {'label':'Upload', 'action': 'manage_uploadForm' }, {'label':'Properties', 'action': 'manage_propertiesForm' }, {'label':'Security', 'action': 'manage_access' }, ) # Create a SecurityInfo for this class. We will use this # in the rest of our class definition to make security # assertions. security = ClassSecurityInfo() security.declareProtected('Change permissions', 'manage_access') security.declareProtected('Change Photo', 'manage_editExtFile') security.declareProtected('Change Photo', 'manage_main') security.declareProtected('Change Photo', 'manage_uploadForm') security.declareProtected('Change Photo', 'manage_file_upload') security.declareProtected('Change Photo', 'PUT') security.declareProtected('FTP access', 'manage_FTPstat') security.declareProtected('FTP access', 'manage_FTPget') security.declareProtected('FTP access', 'manage_FTPlist') security.declareProtected('View', 'is_broken') security.declareProtected('View', 'get_size') security.declareProtected('View', 'getContentType') security.declareProtected('View', '_get_filename') security.declareProtected('View', '__str__') # what do people think they're adding? meta_type = 'ExtFile' # location of the file-repository _repository = ['var','photos'] ################################ # Init method # ################################ def __init__(self, id, title='', content_type='', parent_path=None): """ Initialize a new instance of ExtFile If a extFile is created through FTP, self.absolute_url doesn't work. """ self.id = id self.title = title self.__version__ = __version__ self.filename = [] self.file_uploaded = 0 self.content_type = content_type self._v_parent_path = parent_path ################################ # Public methods # ################################ def __str__(self): return self.index_html() def __len__(self): return 1 HEAD__roles__=None def HEAD(self, REQUEST, RESPONSE): """ """ filename = self._get_filename() try: filesize = os.path.getsize(filename) filemtime = os.path.getmtime(filename) except: raise cant_read_exc, ("%s (%s)" %(self.id, filename)) RESPONSE.setHeader('Last-Modified', rfc1123_date(filemtime)) RESPONSE.setHeader('Content-Length', filesize) RESPONSE.setHeader('Content-Type', self.content_type) return '' def __setstate__(self,state): ExtFile.inheritedAttribute('__setstate__')(self, state) if not hasattr(self,'file_uploaded'): self.file_uploaded = 1 security.declareProtected('View', 'index_html') def index_html(self, REQUEST, RESPONSE): """ Returns the contents of the file. Also, sets the Content-Type HTTP header to the objects content type. """ icon = 0 if icon: filename = join(SOFTWARE_HOME, 'Products', 'Photo', self.getIconPath()) content_type = 'image/gif' else: filename = self._get_filename() content_type = self.content_type if not isfile(filename): self._undo() if not isfile(filename): filename = join(SOFTWARE_HOME, 'Products', 'Photo', 'icons', 'broken.gif') content_type = 'image/gif' self._output_file(REQUEST, RESPONSE, filename, content_type) def _output_file(self, REQUEST, RESPONSE, filename, content_type): """ Write the necesary header information for (possibly) chunked output """ cant_read_exc = "Can't read: " try: filesize = os.path.getsize(filename) filemtime = os.path.getmtime(filename) except: raise cant_read_exc, ("%s (%s)" %(self.id, filename)) # HTTP If-Modified-Since header handling. header=REQUEST.get_header('If-Modified-Since', None) if header is not None: header=string.split(header, ';')[0] # Some proxies seem to send invalid date strings for this # header. If the date string is not valid, we ignore it # rather than raise an error to be generally consistent # with common servers such as Apache (which can usually # understand the screwy date string as a lucky side effect # of the way they parse it). # This happens to be what RFC2616 tells us to do in the face of an # invalid date. try: mod_since=long(DateTime(header).timeTime()) except: mod_since=None if mod_since is not None: last_mod = long(filemtime) if last_mod > 0 and last_mod <= mod_since: # Set header values since apache caching will return Content-Length # of 0 in response if size is not set here RESPONSE.setHeader('Last-Modified', rfc1123_date(filemtime)) RESPONSE.setHeader('Content-Type', content_type) RESPONSE.setHeader('Content-Length', filesize) RESPONSE.setHeader('Accept-Ranges', 'bytes') RESPONSE.setStatus(304) return '' # HTTP Range header handling. Typical header line (folded): # Range:bytes=265089-266112, 257921-265088, 119720-121023, # 121024-151023, 151024-181023, 181024-211023, # 211024-241023, 241024-257920 range = REQUEST.get_header('Range', None) if_range = REQUEST.get_header('If-Range', None) if range is not None: ranges = HTTPRangeSupport.parseRange(range) if if_range is not None: # Only send ranges if the data isn't modified, otherwise send # the whole object. Support both ETags and Last-Modified dates! if len(if_range) > 1 and if_range[:2] == 'ts': # ETag: if if_range != self.http__etag(): # Modified, so send a normal response. We delete # the ranges, which causes us to skip to the 200 # response. ranges = None else: # Date date = string.split(if_range, ';')[0] try: mod_since=long(DateTime(date).timeTime()) except: mod_since=None if mod_since is not None: last_mod = long(filemtime) if last_mod > mod_since: # Modified, so send a normal response. We delete # the ranges, which causes us to skip to the 200 # response. ranges = None if ranges: # Search for satisfiable ranges. satisfiable = 0 for start, end in ranges: if start < filesize: satisfiable = 1 break if not satisfiable: RESPONSE.setHeader('Content-Range', 'bytes */%d' % filesize) RESPONSE.setHeader('Accept-Ranges', 'bytes') RESPONSE.setHeader('Last-Modified', rfc1123_date(filemtime)) RESPONSE.setHeader('Content-Type', content_type) RESPONSE.setHeader('Content-Length', filesize) RESPONSE.setStatus(416) return '' # Can we optimize? #ranges = HTTPRangeSupport.optimizeRanges(ranges, filesize) if len(ranges) == 1: # Easy case, set extra header and return partial set. start, end = ranges[0] size = end - start RESPONSE.setHeader('Last-Modified', rfc1123_date(filemtime)) RESPONSE.setHeader('Content-Type', content_type) RESPONSE.setHeader('Content-Length', size) RESPONSE.setHeader('Accept-Ranges', 'bytes') RESPONSE.setHeader('Content-Range', 'bytes %d-%d/%d' % (start, end - 1, filesize)) RESPONSE.setStatus(206) # Partial content fd = open(filename,'rb') self._writesegment(fd, RESPONSE,start,size) fd.close() return '' else: # Ignore multi-part ranges for now, pretend we don't know # about ranges at all. # When we get here, ranges have been optimized, so they are # in order, non-overlapping, and start and end values are # positive integers. boundary = choose_boundary() # Calculate the content length size = (8 + len(boundary) + # End marker length len(ranges) * ( # Constant lenght per set 49 + len(boundary) + len(content_type) + len('%d' % filesize))) for start, end in ranges: # Variable length per set size = (size + len('%d%d' % (start, end - 1)) + end - start) RESPONSE.setHeader('Content-Length', size) RESPONSE.setHeader('Accept-Ranges', 'bytes') RESPONSE.setHeader('Last-Modified', rfc1123_date(filemtime)) RESPONSE.setHeader('Content-Type', 'multipart/byteranges; boundary=%s' % boundary) RESPONSE.setStatus(206) # Partial content pos = 0 fd = open(filename,'rb') for start, end in ranges: RESPONSE.write('\r\n--%s\r\n' % boundary) RESPONSE.write('Content-Type: %s\r\n' % content_type) RESPONSE.write( 'Content-Range: bytes %d-%d/%d\r\n\r\n' % ( start, end - 1, filesize)) self._writesegment(fd, RESPONSE,start,end-start) RESPONSE.write('\r\n--%s--\r\n' % boundary) fd.close() return '' RESPONSE.setHeader('Last-Modified', rfc1123_date(filemtime)) RESPONSE.setHeader('Content-Type', content_type) RESPONSE.setHeader('Content-Length', filesize) RESPONSE.setHeader('Accept-Ranges', 'bytes') self._copy(filename, RESPONSE) return '' def view_image_or_file(self): """ The default view of the contents of the File or Image. """ raise 'Redirect', self.absolute_url() security.declareProtected('View', 'link') def link(self, text='', **args): """ return a HTML link tag to the file """ if text=='': text = self.title_or_id() strg = '' % self.absolute_url() def is_broken(self): """ Check if external file exists and return true (1) or false (0) """ fn = self._get_filename() if not isfile(fn): self._undo() if not isfile(fn): return 1 return 0 def get_size(self): """ Returns the size of the file or image """ if not self.file_uploaded: return 0 fn = self._get_filename() if not isfile(fn): self._undo() if isfile(fn): size = os.stat(fn)[6] else: size = 0 return size rawsize = get_size # getSize = get_size def size(self): """ Returns a formatted stringified version of the file size """ return self._bytetostring(self.get_size()) def _get_filename(self, filename=None): """ Generate the full filename, including directories from self._repository and self.filename """ if filename == None: filename=self.filename path = INSTANCE_HOME for item in self._repository: path = join(path,item) if type(filename)==types.ListType: for item in filename: path = join(path,item) elif filename!='': path = join(path,filename) return path ################################ # Protected management methods # ################################ # Management Interface manage_main = DTMLFile('dtml/extFileEdit', globals()) def manage_editExtFile(self, title='', content_type='', REQUEST=None): """ Manage the edited values """ if content_type != '': self.content_type = content_type if self.title != title: self.title = title if REQUEST is not None: message = "The properties of %s have been changed!" % self.id return self.manage_main(self,REQUEST,manage_tabs_message=message) # File upload Interface manage_uploadForm = DTMLFile('dtml/docUpload', globals()) def manage_file_upload(self, file='', content_type='', REQUEST=None): """ Upload file from local directory """ new_fn = self._get_ufn(self.filename) self._copy(file, self._get_filename(new_fn)) try: file.seek(0) except: pass content = file.read() self.content_type = self._get_content_type(file, content[:100], self.id, content_type or self.content_type) self.filename = new_fn file.close() self.file_uploaded = 1 if REQUEST is not None: message = "The file was uploaded successfully!" return self.manage_main(self,REQUEST,manage_tabs_message=message) manage_FTPget = index_html def PUT(self, REQUEST, RESPONSE): """ Handle HTTP/FTP PUT requests """ self.dav__init(REQUEST, RESPONSE) self.dav__simpleifhandler(REQUEST, RESPONSE) content_type = REQUEST.get_header('content-type', None) instream = REQUEST['BODYFILE'] new_fn = self._get_ufn(self.filename) self._copy(instream, self._get_filename(new_fn)) try: instream.seek(0) except: pass self.file_uploaded = 1 try: self.content_type = self._get_content_type(instream, instream.read(100), self.id, content_type or self.content_type) except: if UNDO_POLICY==ALWAYS_BACKUP: os.remove(self._get_filename(new_fn)) self.file_uploaded = 0 else: self.filename = new_fn RESPONSE.setStatus(204) return RESPONSE ################################ # Private methods # ################################ def _get_content_type(self, file, body, id, content_type=None): """ Determine the mime-type """ headers = getattr(file, 'headers', None) if headers and headers.has_key('content-type'): content_type = headers['content-type'] else: if type(body) is not type(''): body = body.data content_type, enc = guess_content_type(getattr(file,'filename',id), body, content_type) return content_type def _writesegment(self, instream, outstream, start, length): """ Write a segment of the file. The file is already open. """ instream.seek(start) blocksize = 2<<16 if length < blocksize: nextsize = length else: nextsize = blocksize try: while length > 0: block = instream.read(nextsize) outstream.write(block) length = length - len(block) if length < blocksize: nextsize = length else: nextsize = blocksize except IOError: raise IOError, ("%s (%s)" %(self.id, filename)) def _copy(self, infile, outfile, maxblocks=16384): """ Read binary data from infile and write it to outfile infile and outfile may be strings, in which case a file with that name is opened, or filehandles, in which case they are accessed directly. A block is 131072 bytes. maxblocks prevents it from >2GB """ if type(infile) is types.StringType: try: instream = open(infile, 'rb') except IOError: self._undo() try: instream = open(infile, 'rb') except IOError: raise IOError, ("%s (%s)" %(self.id, infile)) close_in = 1 else: instream = infile close_in = 0 if type(outfile) is types.StringType: try: outstream = open(outfile, 'wb') except IOError: raise IOError, ("%s (%s)" %(self.id, outfile)) close_out = 1 else: outstream = outfile close_out = 0 try: blocksize = 2<<16 block = instream.read(blocksize) outstream.write(block) maxblocks = maxblocks - 1 while len(block)==blocksize and maxblocks > 0: maxblocks = maxblocks - 1 block = instream.read(blocksize) outstream.write(block) except IOError: raise IOError, ("%s (%s)" %(self.id, filename)) if close_in: instream.close() if close_out: outstream.close() def _bytetostring (self, value): """ Convert an int-value (file-size in bytes) to an String with the file-size in Byte, KB or MB """ bytes = float(value) if bytes >= 1000: bytes = bytes/1024 typ = 'KB' if bytes >= 1000: bytes = bytes/1024 typ = 'MB' strg = '%4.2f' % bytes strg = strg[:4] if strg[3]=='.': strg = strg[:3] else: typ = 'Bytes' strg = '%4.0f' % bytes strg = strg+ ' ' + typ return strg def _undo (self): """ restore filename after undo or copy-paste """ if self.filename == []: return fn = self._get_filename() if not isfile(fn) and isfile(fn+'.undo'): self._restorefile(fn) self.file_uploaded = 1 def _deletefile(self, filename): """ Move the file to the undo file """ try: os.rename(filename, filename + '.undo') except: pass self.file_uploaded = 0 def _copyfile(self, old_fn, new_fn): """ Copy a file in the repository This actually involves copying two files """ self._copy(old_fn, new_fn) def _restorefile(self, filename): """ Recover the file from undo """ try: os.rename(filename + '.undo', filename) except: pass def _get_ufn(self, filename): """ If no unique filename has been generated, generate one otherwise, return the existing one. """ if UNDO_POLICY==ALWAYS_BACKUP or filename==[]: new_fn = self._get_new_ufn() else: new_fn = filename[:] if filename: old_fn = self._get_filename(filename) if UNDO_POLICY==ALWAYS_BACKUP: self._deletefile(old_fn) else: self._restorefile(old_fn) return new_fn def _get_new_ufn(self): """ Create a new unique filename """ # We can get in the situation that the object is floating in memory # when the upload is taking place. In that case we don't have a # absolute_url, and must rely on _v_parent_path - set in __init__ if hasattr(self,'_v_parent_path') and self._v_parent_path: rel_url_list = string.split(self._v_parent_path, '/') else: rel_url_list = string.split(self.absolute_url(1), '/')[:-1] rel_url_list = filter(None, rel_url_list) pos = string.rfind(self.id, '.') if (pos+1): id_name = self.id[:pos] id_ext = self.id[pos:] else: id_name = self.id id_ext = '' # generate directory structure dirs = [] if REPOSITORY==SYNC_ZODB: dirs = rel_url_list elif REPOSITORY==SLICED: slice_depth = 2 # modify here, if you want a different slice depth slice_width = 1 # modify here, if you want a different slice width temp = id_name for i in range(slice_depth): if len(temp)=0: fileformat = string.replace(fileformat, "%t", "%c") counter = int(DateTime().strftime('%m%d%H%M%S')) else: counter = 0 invalid_format_exc = "Invalid file format: " if string.find(fileformat, "%c")==-1: raise invalid_format_exc, FILE_FORMAT # user (%u) if string.find(fileformat, "%u")>=0: if (hasattr(self, "REQUEST") and self.REQUEST.has_key('AUTHENTICATED_USER')): user = self.REQUEST['AUTHENTICATED_USER'].name fileformat = string.replace(fileformat, "%u", user) else: fileformat = string.replace(fileformat, "%u", "") # path (%p) if string.find(fileformat, "%p") >= 0: temp = string.joinfields (rel_url_list, "_") fileformat = string.replace(fileformat, "%p", temp) # file and extension (%n and %e) if string.find(fileformat,"%n") >= 0 or string.find(fileformat,"%e") >= 0: fileformat = string.replace(fileformat, "%n", id_name) fileformat = string.replace(fileformat, "%e", id_ext) # make the directories path = self._get_filename(dirs) if not os.path.isdir(path): mkdir_exc = "Can't create directory: " try: os.makedirs(path) except: raise mkdir_exc, path # search for unique filename if counter: fn = join(path, string.replace(fileformat, "%c", `counter`)) else: fn = join(path, string.replace(fileformat, "%c", '')) while (isfile(fn) or isfile(fn+'.undo')): counter = counter+1 fn = join(path, string.replace(fileformat, "%c", `counter`)) if counter: fileformat = string.replace(fileformat, "%c", `counter`) else: fileformat = string.replace(fileformat, "%c", '') dirs.append(fileformat) return dirs ################################ # Special management methods # ################################ def manage_afterClone(self, item): """ When a copy of the object is created (zope copy-paste-operation), this function is called by CopySupport.py. A copy of the external file is created and self.filename is changed. Called AFTER manage_afterAdd """ old_fn = self._get_filename() new_fn = self._get_new_ufn() if isfile(old_fn): self._copyfile(old_fn,self._get_filename(new_fn)) self.filename = new_fn self.file_uploaded = 1 return ExtFile.inheritedAttribute ("manage_afterClone") (self, item) def manage_afterAdd(self, item, container, new_fn=None): """ This method is called, whenever _setObject in ObjectManager gets called. This is the case after a normal add and if the object is a result of cut-paste- or rename-operation. In the first case, the external files doesn't exist yet, otherwise it was renamed to .undo by manage_beforeDelete before and must be restored by _undo(). When FTP PUT: __init__, PUT, _setObject called and file_uploaded == 1, filename!=[] When cut-n-paste: file_uploaded set to 0 in manage_beforeDelete and file is renamed to x.undo. Filename points to original file. When copy-n-paste: file_uploaded == 1, and filename points to original file. """ if self.file_uploaded == 0 and self.filename != []: old_fn = self._get_filename() new_fn = new_fn or self._get_new_ufn() if isfile(old_fn): self._copyfile(old_fn,self._get_filename(new_fn)) else: if isfile(old_fn + '.undo'): self._copyfile(old_fn + '.undo', self._get_filename(new_fn)) self.filename = new_fn self.file_uploaded = 1 return ExtFile.inheritedAttribute ("manage_afterAdd") \ (self, item, container) def manage_beforeDelete(self, item, container): """ This method is called, when the object is deleted. To support undo-functionality and because this happens too, when the object is moved (cut-paste) or renamed, the external file is not deleted. It are just renamed to filename.undo and remains in the repository, until it is deleted manually. """ fn = self._get_filename() self._deletefile(fn) self.file_uploaded = 0 return ExtFile.inheritedAttribute ("manage_beforeDelete") \ (self, item, container) Globals.InitializeClass(ExtFile)