Transition from file streams to descriptors for streaming features.

shogun-toolbox · Jul 19, 2011 · 9c42aa2 · 9c42aa2
2 parents 2a8980b + 48366d4
commit 9c42aa2
Show file tree

Hide file tree

Showing 6 changed files with 311 additions and 182 deletions.
diff --git a/src/shogun/io/IOBuffer.cpp b/src/shogun/io/IOBuffer.cpp
@@ -18,65 +18,76 @@
 
 using namespace shogun;
 
+CIOBuffer::CIOBuffer()
+{
+	init();
+}
+
+CIOBuffer::CIOBuffer(int fd)
+{
+	init();
+	working_file = fd;
+}
+
+CIOBuffer::~CIOBuffer()
+{
+	free(space.begin);
+}
+
 void CIOBuffer::init()
 {
 	size_t s = 1 << 16;
-	reserve(space, s);
+	space.reserve(s);
 	endloaded = space.begin;
 }
 
-int CIOBuffer::open_file(const char* name, int flag)
+void CIOBuffer::use_file(int fd)
+{
+	working_file = fd;
+}
+
+int CIOBuffer::open_file(const char* name, char flag)
 {
 	int ret=1;
 	switch(flag){
-	case READ:
-		working_file = fopen(name, "r");
+	case 'r':
+		working_file = open(name, O_RDONLY|O_LARGEFILE);
 		break;
 
-	case WRITE:
-		working_file = fopen(name, "w");
+	case 'w':
+		working_file = open(name, O_WRONLY|O_LARGEFILE);
 		break;
 
 	default:
-		SG_ERROR("Unknown file operation. Something other than READ/WRITE specified.\n");
+		SG_ERROR("Unknown file operation. Something other than 'r'/'w' specified.\n");
 		ret = 0;
 	}
 	return ret;
 }
 
 void CIOBuffer::reset_file()
 {
-	rewind(working_file);
+	lseek(working_file, 0, SEEK_SET);
 	endloaded = space.begin;
 	space.end = space.begin;
 }
 
-CIOBuffer::CIOBuffer()
-{
-	init();
-}
-
-CIOBuffer::~CIOBuffer()
-{
-	free(space.begin);
-}
-
 void CIOBuffer::set(char *p)
 {
 	space.end = p;
 }
 
 ssize_t CIOBuffer::read_file(void* buf, size_t nbytes)
 {
-	return fread(buf, 1, nbytes, working_file);
+	return read(working_file, buf, nbytes);
 }
 
 size_t CIOBuffer::fill()
 {
 	if (space.end_array - endloaded == 0)
 	{
 		size_t offset = endloaded - space.begin;
-		reserve(space, 2 * (space.end_array - space.begin));
+		space.reserve(2 * (space.end_array - space.begin));
 		endloaded = space.begin+offset;
 	}
 	ssize_t num_read = read_file(endloaded, space.end_array - endloaded);
@@ -91,28 +102,34 @@ size_t CIOBuffer::fill()
 
 ssize_t CIOBuffer::write_file(const void* buf, size_t nbytes)
 {
-	return fwrite(buf, 1, nbytes, working_file);
+	return write(working_file, buf, nbytes);
 }
 
 void CIOBuffer::flush()
 {
 	if (write_file(space.begin, space.index()) != (int) space.index())
 		SG_ERROR("Error, failed to write example!\n");
 	space.end = space.begin;
-	fflush(working_file);
+	fsync(working_file);
 }
 
 bool CIOBuffer::close_file()
 {
-	if (working_file == NULL)
+	if (working_file < 0)
 		return false;
 	else
-		return bool(fclose(working_file));
+	{
+		int r = close(working_file);
+		if (r < 0)
+			SG_ERROR("Error closing the file!\n");
+		return true;
+	}
 }
 
-size_t CIOBuffer::readto(char* &pointer, char terminal)
+ssize_t CIOBuffer::readto(char* &pointer, char terminal)
 {
-//Return a pointer to the bytes before the terminal.  Must be less than the buffer size.
+//Return a pointer to the bytes before the terminal.  Must be less
+//than the buffer size.
 	pointer = space.end;
 	while (pointer != endloaded && *pointer != terminal)
 		pointer++;

diff --git a/src/shogun/io/IOBuffer.h b/src/shogun/io/IOBuffer.h
@@ -30,8 +30,6 @@
 #define O_LARGEFILE 0
 #endif
 
-using namespace std;
-
 namespace shogun
 {
 /** @brief An I/O buffer class.
@@ -49,116 +47,136 @@ namespace shogun
 
  public:
 
-	/** 
+	/**
+	 * Constructor.
+	 */
+	CIOBuffer();
+
+	/**
+	 * Constructor taking file descriptor as parameter
+	 *
+	 * @param fd file descriptor to use
+	 */
+	CIOBuffer(int fd);
+
+	/**
+	 * Destructor.
+	 */
+	~CIOBuffer();
+
+	/**
 	 * Initialize the buffer, reserve 64K memory by default.
-	 * 
 	 */
 	void init();
 
-	/** 
+	/**
+	 * Uses the passed file descriptor
+	 *
+	 * @param fd file descriptor to use
+	 */
+	virtual void use_file(int fd);
+
+	/**
 	 * Open a file, in read or write mode.
-	 * 
+	 *
 	 * @param name File name.
-	 * @param flag CIOBuffer::READ or CIOBuffer::WRITE
-	 * 
+	 * @param flag 'r' or 'w'
+	 *
 	 * @return 1 on success, 0 on error.
 	 */
-	virtual int open_file(const char* name, int flag=READ);
+	virtual int open_file(const char* name, char flag='r');
 
-	/** 
+	/**
 	 * Seek back to zero, reset the buffer markers.
-	 * 
 	 */
 	virtual void reset_file();
 
-	/** 
-	 * Constructor.
-	 * 
-	 */
-	CIOBuffer();
-
-	/** 
-	 * Destructor.
-	 * 
-	 */
-	~CIOBuffer();
-
-	/** 
+	/**
 	 * Set the buffer marker to a position.
-	 * 
+	 *
 	 * @param p Character pointer to which the end of buffer space is assigned.
 	 */
 	void set(char *p);
 
-	/** 
+	/**
 	 * Read some bytes from the file into memory.
-	 * 
+	 *
 	 * @param buf void* buffer into which to read.
 	 * @param nbytes number of bytes to read
-	 * 
+	 *
 	 * @return Number of bytes read successfully.
 	 */
 	virtual ssize_t read_file(void* buf, size_t nbytes);
 
-	/** 
+	/**
 	 * Fill the buffer by reading as many bytes from the file as required.
-	 * 
+	 *
 	 * @return Number of bytes read.
 	 */
 	size_t fill();
 
-	/** 
+	/**
 	 * Write to the file from memory.
-	 * 
+	 *
 	 * @param buf void* buffer from which to write.
 	 * @param nbytes Number of bytes to write.
-	 * 
+	 *
 	 * @return Number of bytes successfully written.
 	 */
 	virtual ssize_t write_file(const void* buf, size_t nbytes);
 
-	/** 
+	/**
 	 * Flush the stream; commit all operations to file.
-	 * 
 	 */
 	virtual void flush();
 
-	/** 
+	/**
 	 * Close the file.
-	 * 
-	 * 
+	 *
 	 * @return true on success, false otherwise.
 	 */
 	virtual bool close_file();
 
-	/** 
+	/**
 	 * Reads upto a terminal character from the buffer.
-	 * 
+	 *
 	 * @param pointer Start of the string in the buffer, set by reference.
 	 * @param terminal Terminal character upto which to read.
-	 * 
+	 *
 	 * @return Number of characters read.
 	 */
-	size_t readto(char* &pointer, char terminal);
+	ssize_t readto(char* &pointer, char terminal);
+
+	/**
+	 * Reads upto a newline character from the buffer.
+	 *
+	 * @param pointer Start of the string, set by reference
+	 *
+	 * @return Number of characters read.
+	 */
+	inline ssize_t read_line(char* &pointer)
+	{
+		return readto(pointer, '\n');
+	}
 
 	virtual const char* get_name() const
 	{
 		return "IOBuffer";
 	}
-	
+
 
 public:
-
-	v_array<char> space; /**< space.begin = beginning of loaded
-			      * values.  space.end = end of read or
-			      * written values */
-
-	char* endloaded; 	/**< end of loaded values */
 
-	FILE* working_file;
-	static const int READ = 1;
-	static const int WRITE = 2;
+	/// buffer space
+	v_array<char> space;
+	/* space.begin = beginning of loaded values
+	 * space.end   = end of read or written values */
+
+	/// end of loaded values
+	char* endloaded;
 
+	/// file descriptor
+	int working_file;
 
 };
 }