FileWriteChannel writeChannel throwing java.io.IOException?
12 views
Skip to first unread message
Brandon Donnelson
May 27, 2011, 11:09:59 AM5/27/11
to gwt-goo...@googlegroups.com
I can write to the blobstore smaller files, but when I try larger writes I'm getting errrors (done in task). I'm not sure as to why I'm getting FileWriteChannel writeChannel throwing "java.io.IOException" yet. Its a bit vague on its reason why theres a problem. Is there a time limit thats not being traced?
I decode on the fly and write to blobstore. I am having good success with smaller to medium <5MB files, but when I go larger, streaming the writes are causing problem above. This works perfectly on the dev side. But I'm wondering if there is a time limit in the filewrite not showing up. This is done in a task.
My class thats throwing the error:
package org.gonevertical.core.server.jdo.data.base64;
import java.io.ByteArrayOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.gonevertical.core.client.ui.admin.blobs.BlobData;
import org.gonevertical.core.client.ui.admin.blobs.FileByteData;
import org.gonevertical.core.client.ui.admin.blobs.file.FileData;
import org.gonevertical.core.client.ui.admin.thingstuff.ThingStuffData;
import org.gonevertical.core.client.ui.admin.thingstufftype.ThingStuffTypeData;
import org.gonevertical.core.client.ui.admin.thingtype.ThingTypeData;
import org.gonevertical.core.server.ServerPersistence;
import org.gonevertical.core.server.db.Db_Thing;
import org.gonevertical.core.server.jdo.data.ThingStuffJdo;
import com.google.appengine.api.blobstore.BlobKey;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.PreparedQuery;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.FilterOperator;
import com.google.appengine.api.datastore.Text;
import com.google.appengine.api.files.AppEngineFile;
import com.google.appengine.api.files.FileService;
import com.google.appengine.api.files.FileServiceFactory;
import com.google.appengine.api.files.FileWriteChannel;
import com.google.appengine.api.files.FinalizationException;
import com.google.appengine.api.files.LockException;
public class WriteBase64 {
private static final Logger log = Logger.getLogger(WriteBase64.class.getName());
private final char[] BASE64ALPHA = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/".toCharArray();
private int[] toInt = new int[128];
private ServerPersistence sp;
private FileByteData fbd;
private AppEngineFile file;
private FileWriteChannel writeChannel;
private FileService fileService;
private long newfileThingId;
private String filename;
private boolean success;
public WriteBase64(ServerPersistence sp) {
this.sp = sp;
for (int i=0; i < BASE64ALPHA.length; i++){
toInt[BASE64ALPHA[i]] = i;
}
}
public boolean decodeToBlob(FileByteData fbd) {
this.fbd = fbd;
if (fbd == null) {
return false;
}
// init the filewriting
success = initBlobWrite();
if (success == false) {
return false;
}
decode();
if (success == false) {
return false;
}
// close and create a newFileThingId
close();
return success;
}
public long getThingId() {
return newfileThingId;
}
private void close() {
try {
writeChannel.closeFinally();
} catch (IllegalStateException e) {
log.severe("close(): Error 6:" + e.toString());
e.printStackTrace();
} catch (IOException e) {
log.severe("close(): Error 7:" + e.toString());
e.printStackTrace();
}
BlobKey blobKey = fileService.getBlobKey(file);
if (blobKey == null) {
log.severe("close(): Blobkey was null. trying workaround.");
blobKey = tryfindingBlobKey();
if (blobKey == null) {
success = false;
log.severe("close(): Blobkey was null.");
return;
}
}
// create file thing
Db_Thing dbT = new Db_Thing(sp);
long fid = dbT.createThing(ThingTypeData.TYPE_BLOBS);
setNewFileThingId(fid);
// link file to parentthingid
ThingStuffJdo tsj = new ThingStuffJdo(sp);
//tsj.saveStuff_ThingLink(fbd.getLinkTypeId(), fbd.getLinkToThingId(), fid); // this can't be done here, due to not saving the links
BlobData blobData = new BlobData();
blobData.setKey(blobKey.getKeyString());
FileData fileData = new FileData();
fileData.setBlobData(blobData);
ThingStuffData tsd = new ThingStuffData();
tsd.setParentThingId(fid);
tsd.setStuffTypeId(ThingStuffTypeData.THINGSTUFFTYPE_BLOB);
tsd.setFileData(fileData);
tsj.saveUniqueStuffTypeIdForParentThingId(tsd);
//log.info("newFileThingId=" + newfileThingId);
}
private BlobKey tryfindingBlobKey() {
// TODO!!!!
return null;
}
private void setNewFileThingId(long fid) {
this.newfileThingId = fid;
}
private boolean initBlobWrite() {
if (fbd == null) {
return false;
}
// WORKAROUND - this will help, when blobkey is found to be null
filename = fbd.getFileName();
//filename = StringUtils.getRandomString(5) + "_" + filename;
fileService = FileServiceFactory.getFileService();
try {
file = fileService.createNewBlobFile(fbd.getContentType(), filename);
success = true;
log.info("initBlobWrite(): INFO: success in setting up file.");
} catch (IOException e) {
success = false;
log.severe("initBlobWrite(): Error 1:" + e.toString());
e.printStackTrace();
}
boolean lock = true;
try {
writeChannel = fileService.openWriteChannel(file, lock);
success = true;
log.info("initBlobWrite(): INFO: success in setting up writechannel.");
} catch (FileNotFoundException e) {
success = false;
log.severe("initBlobWrite(): Error 2:" + e.toString());
e.printStackTrace();
} catch (FinalizationException e) {
success = false;
log.severe("initBlobWrite(): Error 2.5:" + e.toString());
e.printStackTrace();
} catch (LockException e) {
success = false;
log.severe("initBlobWrite(): Error 3:" + e.toString());
e.printStackTrace();
} catch (IOException e) {
success = false;
log.severe("initBlobWrite(): Error 4:" + e.toString());
e.printStackTrace();
}
return success;
}
/**
* query the chunks, start to middle chunks must be divisible by 4, except end
*
* @return
*/
private void decode() {
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
try {
Query q = new Query("BlobTmpJdo");
q.addFilter("key", FilterOperator.EQUAL, fbd.getKey());
q.addSort("index");
PreparedQuery e = datastore.prepare(q);
Iterator<Entity> itr = e.asIterator();
while (itr.hasNext()) {
Entity entity = itr.next();
Text tb64 = (Text) entity.getProperty("base64");
if (tb64 != null) {
String b64 = tb64.getValue();
//log.info("query(): index=" + entity.getProperty("index") + " base64=" + tb64.getValue());
decode(b64);
if (success == false) { // quit going forward if a process has failed
// break; // TODO enable in the future
}
} else {
// TODO deal with error
log.warning("query(): index=" + entity.getProperty("index") + " base64=" + tb64);
}
}
} catch (Exception e) {
success = false;
log.log(Level.SEVERE, "", e);
e.printStackTrace();
}
}
private void decode(String b64) {
ByteArrayOutputStream out = null;
try {
out = null;
int mask = 0xFF;
int index = 0;
for (int i=0; i < b64.length(); i+=4) {
if (out == null) {
out = new ByteArrayOutputStream();
}
int c0 = toInt[b64.charAt(i)]; // char 1
int c1 = toInt[b64.charAt(i + 1)]; // char 2
byte b1 = (byte) (((c0 << 2) | (c1 >> 4)) & mask);
out.write(b1);
index++;
if (index >= b64.length()) {
break;
}
int c2 = toInt[b64.charAt(i + 2)]; // char 3
byte b2 = (byte) (((c1 << 4) | (c2 >> 2)) & mask);
out.write(b2);
index++;
if (index >= b64.length()) {
break;
}
int c3 = toInt[b64.charAt(i + 3)]; // char 4
byte b3 = (byte) (((c2 << 6) | c3) & mask);
out.write(b3);
}
} catch (Exception e) {
success = false;
log.severe("decode(b64): ERROR: can't decode base64.");
return;
}
if (out != null) {
writeToBlob(out);
}
}
private void writeToBlob(ByteArrayOutputStream out) {
if (out == null) {
log.info("writeToBlob(): INFO: ByteArrayOutputStream is null.");
return;
}
if (out.toByteArray() == null) {
log.info("writeToBlob(): INFO: ByteArrayOutputStream out.toByteArray() is null.");
return;
}
if (out.toByteArray().length == 0) {
log.info("writeToBlob(): INFO: ByteArrayOutputStream out.toByteArray().length is 0.");
return;
}
ByteBuffer bb = ByteBuffer.wrap(out.toByteArray());
try {
writeChannel.write(bb);
log.info("writeToBlob(): INFO: success in writing bytes");
} catch (IOException e) {
success = false;
log.severe("writeToBlob(): Error 5: " + e.toString());
//e.printStackTrace();
}
}
}
Brandon Donnelson
Brandon Donnelson
May 27, 2011, 11:12:58 AM5/27/11
to gwt-goo...@googlegroups.com
The error:
E2011-05-27 07:41:59.139
org.gonevertical.core.server.jdo.data.base64.WriteBase64 writeToBlob: writeToBlob(): Error 5: java.io.IOException
E2011-05-27 07:41:59.149
org.gonevertical.core.server.jdo.data.base64.WriteBase64 decode: com.google.appengine.api.datastore.DatastoreTimeoutException: The datastore operation timed out, or the data was temporarily unavailable. at com.google.appengine.api.datastore.DatastoreApiHelper.translateError(DatastoreApiHelper.java:46) at com.google.appengine.api.datastore.DatastoreApiHelper$1.convertException(DatastoreApiHelper.java:98) at com.google.appengine.api.utils.FutureWrapper.get(FutureWrapper.java:69) at com.google.appengine.api.datastore.FutureHelper.getInternal(FutureHelper.java:71) at com.google.appengine.api.datastore.FutureHelper.quietGet(FutureHelper.java:32) at com.google.appengine.api.datastore.QueryResultsSourceImpl.loadMoreEntities(QueryResultsSourceImpl.java:69) at com.google.appengine.api.datastore.QueryResultsSourceImpl.loadMoreEntities(QueryResultsSourceImpl.java:56) at com.google.appengine.api.datastore.QueryResultIteratorImpl.ensureLoaded(QueryResultIteratorImpl.java:156) at com.google.appengine.api.datastore.QueryResultIteratorImpl.hasNext(QueryResultIteratorImpl.java:65) at org.gonevertical.core.server.jdo.data.base64.WriteBase64.decode(WriteBase64.java:219) at org.gonevertical.core.server.jdo.data.base64.WriteBase64.decodeToBlob(WriteBase64.java:87) at org.gonevertical.core.server.jdo.data.BlobTmpJdo.writeTmpBlob(BlobTmpJdo.java:335) at org.gonevertical.core.server.jdo.data.BlobTmpJdo.writeTmpBlob(BlobTmpJdo.java:321) at org.gonevertical.core.server.servlet.Servlet_Task.startWriteToBlob(Servlet_Task.java:106) at org.gonevertical.core.server.servlet.Servlet_Task.doGet(Servlet_Task.java:92) at javax.servlet.http.HttpServlet.service(HttpServlet.java:617) at javax.servlet.http.HttpServlet.service(HttpServlet.java:717) at org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1166) at com.google.apphosting.utils.servlet.ParseBlobUploadFilter.doFilter(ParseBlobUploadFilter.java:97) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157) at com.google.apphosting.runtime.jetty.SaveSessionFilter.doFilter(SaveSessionFilter.java:35) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157) at com.google.apphosting.utils.servlet.TransactionCleanupFilter.doFilter(TransactionCleanupFilter.java:43) at org.mortbay.jetty.servlet.ServletHandler$CachedChain.doFilter(ServletHandler.java:1157) at org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:388) at org.mortbay.jetty.security.SecurityHandler.handle(SecurityHandler.java:216) at org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) at org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765) at org.mortbay.jetty.webapp.WebAppContext.handle(WebAppContext.java:418) at com.google.apphosting.runtime.jetty.AppVersionHandlerMap.handle(AppVersionHandlerMap.java:238) at org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) at org.mortbay.jetty.Server.handle(Server.java:326) at org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) at org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:923) at com.google.apphosting.runtime.jetty.RpcRequestParser.parseAvailable(RpcRequestParser.java:76) at org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) at com.google.apphosting.runtime.jetty.JettyServletEngineAdapter.serviceRequest(JettyServletEngineAdapter.java:135) at com.google.apphosting.runtime.JavaRuntime.handleRequest(JavaRuntime.java:260) at com.google.apphosting.base.RuntimePb$EvaluationRuntime$2.handleRequest(RuntimePb.java:9669) at com.google.net.rpc.impl.RpcUtil.runRpcInApplication(RpcUtil.java:439) at com.google.net.rpc.impl.Server$RpcTask.runInContext(Server.java:573) at com.google.tracing.TraceContext$TraceContextRunnable$1.run(TraceContext.java:448) at com.google.tracing.TraceContext.runInContext(TraceContext.java:688) at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContextNoUnref(TraceContext.java:326) at com.google.tracing.TraceContext$AbstractTraceContextCallback.runInInheritedContext(TraceContext.java:318) at com.google.tracing.TraceContext$TraceContextRunnable.run(TraceContext.java:446) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1110) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:603) at java.lang.Thread.run(Thread.java:636)
Brandon Donnelson
May 27, 2011, 3:31:55 PM5/27/11
to gwt-goo...@googlegroups.com
Oops, wrong group, my bad.
Reply all
Reply to author
Forward
0 new messages